dolphinscheduler最新版2.0.2+mysql8.0.16部署（一）

1. 下载dolphinscheduler

下载地址https://dolphinscheduler.apache.org/en-us/download/download.html

下载版本为：src源码，需要重新编译

查看pom.xml对应的组件版本，这里不介绍编译方式，下次用到介绍

  1 4.3.0
  2 
  3 3.4.14
  4 
  5 5.3.12
  6 
  7 2.5.6
  8 
  9 1.8
 10 
 11 1.2.3
 12 
 13 2.7.3
 14 
 15 2.3.0
 16 
 17 2.10.5
 18 
 19 3.2.0
 20 
 21 2.0.1
 22 
 23 9.1.3
 24 
 25 1.2.4
 26 
 27 1.4.200
 28 
 29 1.11
 30 
 31 1.1.1
 32 
 33 4.4.1
 34 
 35 4.4.1
 36 
 37 4.12
 38 
 39 8.0.16
 40 
 41 1.7.5
 42 
 43 1.7.5
 44 
 45 3.2.2
 46 
 47 3.0.1
 48 
 49 1.9.4
 50 
 51 1.10
 52 
 53 2.6
 54 
 55 1.5
 56 
 57 4.1.2
 58 
 59 3.1.0
 60 
 61 4.1
 62 
 63 24.1-jre
 64 
 65 42.2.5
 66 
 67 2.1.0
 68 
 69 2.4
 70 
 71 3.9.1
 72 
 73 0.1.52
 74 
 75 6.1.0.jre8
 76 
 77 0.238.1
 78 
 79 3.1.12
 80 
 81 3.1.2
 82 
 83 3.4.14
 84 
 85 2.12.0
 86 
 87 1.6
 88 
 89 3.3
 90 
 91 3.1.0
 92 
 93 2.5.3
 94 
 95 2.10.3
 96 
 97 2.4
 98 
 99 2.22.1
100 
101 3.1.1
102 
103 2.2.0
104 
105 0.8.4
106 
107 1.0
108 
109 false
110 
111 2.7
112 
113 2.21.0
114 
115 2.0.2
116 
117 2.5
118 
119 1.9.3
120 
121 2.9.2
122 
123 1.5.24
124 
125 2.0.0
126 
127 1.7.2
128 
129 0.9.12
130 
131 1.9.16
132 
133 1.5.1
134 
135 0.10.9
136 
137 1.0.1
138 
139 false
140 
141 4.1.53.Final
142 
143 3.2.0

以dolphin-server为server机，以下需要在每台机器创建用户2. 创建应用用户及免密

# 创建用户需使用root登录，设置部署用户名，请自行修改，后面以dolphinscheduler为例

useradd dolphin;

# 设置用户密码，请自行修改，后面以dolphinscheduler123为例

echo "dolphin" | passwd --stdin dolphin

# 配置sudo免密

echo 'dolphin  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' >> /etc/sudoers

在84机器上执行

#add ip hostname

vi /etc/hosts

198.28.1.82 ht-28-tdh82

198.28.1.83 ht-28-tdh83

198.28.1.84 ht-28-tdh84

#同步dolphin-server上的/etc/hosts到所有部署机器

for ip in dolphin-82 dolphin-83;    

do

sudo scp -r /etc/hosts  $ip:/etc/          #在运行中需要输入root密码

done

#在dolphin-84上，切换到部署用户并配置ssh本机免密登录

su - dolphin;

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

chmod 600 ~/.ssh/authorized_keys

for ip in ht-28-tdh82 ht-28-tdh83;     #请将此处ds2 ds3替换为自己要部署的机器的hostname

do

ssh-copy-id  $ip   #该操作执行过程中需要手动输入dolphinscheduler用户的密码

done

3. Mysql部署

下载mysql 版本 (mysql-8.0.16-linux-glibc2.12-x86_64.tar)

cd /opt/dolphin

tar -xvf mysql-8.0.16-linux-glibc2.12-x86_64.tar

tar -xvJf mysql-8.0.16-linux-glibc2.12-x86_64.tar.xz

# 建立软链接，便于以后版本升级

ln -s mysql-8.0.16-linux-glibc2.12-x86_64 mysql

创建临时目录、数据目录和日志目录

mkdir -p /opt/dolphin/mysql/3306/data

mkdir -p /opt/dolphin/mysql/3306/log

mkdir -p /opt/dolphin/mysql/3306/tmp

# 创建配置文件-root下

cd /etc

# 在my.cnf文件中添加对应的配置项

cp my.cnf my.cnf.bak20220125
vi my.cnf

 1 [client] # 客户端设置，即客户端默认的连接参数
 2 
 3 port = 3306 # 默认连接端口
 4 
 5 socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 用于本地连接的socket套接字，mysqld守护进程生成了这个文件
 6 
 7 [mysqld] # 服务端基本设置
 8 
 9 # 基础设置
10 
11 server-id = 1 # Mysql服务的唯一编号 每个mysql服务Id需唯一
12 
13 port = 3306 # MySQL监听端口
14 
15 basedir = /opt/dolphin/mysql # MySQL安装根目录
16 
17 datadir = /opt/dolphin/mysql/3306/data # MySQL数据文件所在位置
18 
19 tmpdir = /opt/dolphin/mysql/3306/tmp # 临时目录，比如load data infile会用到
20 
21 socket = /opt/dolphin/mysql/3306/tmp/mysql.sock # 为MySQL客户端程序和服务器之间的本地通讯指定一个套接字文件
22 
23 pid-file = /opt/dolphin/mysql/3306/log/mysql.pid # pid文件所在目录
24 
25 #skip_name_resolve = 1 # 只能用IP地址检查客户端的登录，不用主机名
26 
27 character-set-server = utf8mb4 # 数据库默认字符集,主流字符集支持一些特殊表情符号（特殊表情符占用4个字节）
28 
29 transaction_isolation = READ-COMMITTED # 事务隔离级别，默认为可重复读，MySQL默认可重复读级别
30 
31 collation-server = utf8mb4_general_ci # 数据库字符集对应一些排序等规则，注意要和character-set-server对应
32 
33 init_connect='SET NAMES utf8mb4' # 设置client连接mysql时的字符集,防止乱码
34 
35 lower_case_table_names = 1 # 是否对sql语句大小写敏感，1表示不敏感
36 
37 max_connections = 400 # 最大连接数
38 
39 max_connect_errors = 1000 # 最大错误连接数
40 
41 explicit_defaults_for_timestamp = true # TIMESTAMP如果没有显示声明NOT NULL，允许NULL值
42 
43 max_allowed_packet = 128M # SQL数据包发送的大小，如果有BLOB对象建议修改成1G
44 
45 interactive_timeout = 1800 # MySQL连接闲置超过一定时间后(单位：秒)将会被强行关闭
46 
47 wait_timeout = 1800 # MySQL默认的wait_timeout值为8个小时, interactive_timeout参数需要同时配置才能生效
48 
49 tmp_table_size = 16M # 内部内存临时表的最大值 ，设置成128M；比如大数据量的group by ,order by时可能用到临时表；超过了这个值将写入磁盘，系统IO压力增大

# 进入MySQL的bin目录安装MySQL数据库

cd /opt/dolphin/mysql/bin

# 初始化数据库，并指定启动mysql的用户

./mysqld --initialize --user=dolphin

生成随机密码

root@localhost: ,8v_

设置开机自启动服务

# 复制启动脚本到资源目录

cp /opt/dolphin/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld

# 增加mysqld服务控制脚本执行权限

chmod +x /etc/rc.d/init.d/mysqld

# 将mysqld服务加入到系统服务

chkconfig --add mysqld

# 检查mysqld服务是否已经生效

chkconfig --list mysqld

# 切换至mysql用户，启动mysql

su - dolphin

service mysqld start

配置环境变量

# 修改配置文件，增加

cp .bash_profile .bash_profile_bak20220125

vi .bash_profile

export PATH=$PATH:/opt/dolphin/mysql/bin

# 立即生效

source .bash_profile

登陆，修改密码

# 登陆mysql

./mysql -uroot -p -S/opt/dolphin/mysql/3306/tmp/mysql.sock

报错：ERROR 2059 (HY000): Authentication plugin 'caching_sha2_password' cannot be loaded: /usr/lib64/mysql/plugin/caching_sha2_password.so: cannot open shared object file: No such file or directory

修改

vi /etc/my.cnf

[mysqld]

default_authentication_plugin=mysql_native_password

重启mysql

service mysqld restart

# 修改root用户密码

alter user 'root'@'localhost'IDENTIFIED BY '123456';

4. 数据库初始化

· 进入数据库，默认数据库是PostgreSQL，如选择Mysql的话，后续需要添加mysql-connector-java驱动包到DolphinScheduler的lib目录下，这里以mysql为例

mysql -uroot -p

· 进入数据库命令行窗口后，执行数据库初始化命令，设置访问账号和密码。

mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

Query OK, 1 row affected (0.00 sec)

mysql> create user 'dolphin'@'localhost' identified by 'dolphin';

mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphin'@'localhost';

Query OK, 0 rows affected, 1 warning (0.00 sec)

mysql> flush privileges;

Query OK, 0 rows affected (0.00 sec)

5. Ds部署

root下面创建目录

mkdir -p /opt/dolphin;

赋权

chown -R dolphin:dolphin /opt/dolphin

cd /opt/dolphin;

tar -zxvf apache-dolphinscheduler-2.0.2-bin.tar.gz -C /opt/dolphin;

#在ds1上，修改目录权限，使得部署用户对dolphin-backend目录有操作权限

chown -R dolphin:dolphin apache-dolphinscheduler-2.0.2-bin

· 修改 conf 目录下 application-mysql.yaml 中的下列配置

vi conf/application-mysql.yaml

 1 spring:
 2 
 3 datasource:
 4 
 5 driver-class-name: com.mysql.jdbc.Driver
 6 
 7 url: jdbc:mysql://198.28.1.84:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
 8 
 9 username: dolphin
10 
11 password: dolphin
12 
13 hikari:
14 
15 connection-test-query: select 1
16 
17 minimum-idle: 5
18 
19 auto-commit: true
20 
21 validation-timeout: 3000
22 
23 pool-name: DolphinScheduler
24 
25 maximum-pool-size: 50
26 
27 connection-timeout: 30000
28 
29 idle-timeout: 600000
30 
31 leak-detection-threshold: 0
32 
33 initialization-fail-timeout: 1

还需要手动添加 [?mysql-connector-java 驱动 jar?] 包到 lib 目录下，这里下载的是mysql-connector-java-8.0.16.jar

· 修改并保存完后，执行 script 目录下的创建表及导入基础数据脚本

sh script/create-dolphinscheduler.sh

发现报错，报出来数据库不对为h2，原因是因为初始化数据脚本是按照

/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/conf/config/install_config.conf

去读取配置

可以看下create-dolphinscheduler.sh文件

里面加载了install_config.conf这个文件

  1 #
  2 
  3 # Licensed to the Apache Software Foundation (ASF) under one or more
  4 
  5 # contributor license agreements. See the NOTICE file distributed with
  6 
  7 # this work for additional information regarding copyright ownership.
  8 
  9 # The ASF licenses this file to You under the Apache License, Version 2.0
 10 
 11 # (the "License"); you may not use this file except in compliance with
 12 
 13 # the License. You may obtain a copy of the License at
 14 
 15 #
 16 
 17 # http://www.apache.org/licenses/LICENSE-2.0
 18 
 19 #
 20 
 21 # Unless required by applicable law or agreed to in writing, software
 22 
 23 # distributed under the License is distributed on an "AS IS" BASIS,
 24 
 25 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 26 
 27 # See the License for the specific language governing permissions and
 28 
 29 # limitations under the License.
 30 
 31 #
 32 
 33 # ---------------------------------------------------------
 34 
 35 # INSTALL MACHINE
 36 
 37 # ---------------------------------------------------------
 38 
 39 # A comma separated list of machine hostname or IP would be installed DolphinScheduler,
 40 
 41 # including master, worker, api, alert. If you want to deploy in pseudo-distributed
 42 
 43 # mode, just write a pseudo-distributed hostname
 44 
 45 # Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
 46 
 47 ips="ht-28-tdh82,ht-28-tdh83"
 48 
 49 # Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
 50 
 51 # modify it if you use different ssh port
 52 
 53 sshPort="22"
 54 
 55 # A comma separated list of machine hostname or IP would be installed Master server, it
 56 
 57 # must be a subset of configuration `ips`.
 58 
 59 # Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
 60 
 61 masters="ht-28-tdh84"
 62 
 63 # A comma separated list of machine : or :.All hostname or IP must be a
 64 
 65 # subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
 66 
 67 # Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
 68 
 69 workers="ht-28-tdh82:default,ht-28-tdh83:default"
 70 
 71 # A comma separated list of machine hostname or IP would be installed Alert server, it
 72 
 73 # must be a subset of configuration `ips`.
 74 
 75 # Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
 76 
 77 alertServer="ht-28-tdh84"
 78 
 79 # A comma separated list of machine hostname or IP would be installed API server, it
 80 
 81 # must be a subset of configuration `ips`.
 82 
 83 # Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
 84 
 85 apiServers="ht-28-tdh84"
 86 
 87 # A comma separated list of machine hostname or IP would be installed Python gateway server, it
 88 
 89 # must be a subset of configuration `ips`.
 90 
 91 # Example for hostname: pythonGatewayServers="ds1", Example for IP: pythonGatewayServers="192.168.8.1"
 92 
 93 pythonGatewayServers="ht-28-tdh84"
 94 
 95 # The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
 96 
 97 # Do not set this configuration same as the current path (pwd)
 98 
 99 installPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin"
100 
101 # The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
102 
103 # script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
104 
105 # to be created by this user
106 
107 deployUser="dolphin"
108 
109 # The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
110 
111 dataBasedirPath="/opt/dolphin/apache-dolphinscheduler-2.0.2-bin/tmp"
112 
113 # ---------------------------------------------------------
114 
115 # DolphinScheduler ENV
116 
117 # ---------------------------------------------------------
118 
119 # JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
120 
121 # and this configuration only support one parameter so far.
122 
123 javaHome="/usr/java/jdk1.8.0_144"
124 
125 # DolphinScheduler API service port, also this is your DolphinScheduler UI component's URL port, default value is 12345
126 
127 apiServerPort="12345"
128 
129 # ---------------------------------------------------------
130 
131 # Database
132 
133 # NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
134 
135 # ---------------------------------------------------------
136 
137 # The type for the metadata database
138 
139 # Supported values: ``postgresql``, ``mysql`, `h2``.
140 
141 DATABASE_TYPE=${DATABASE_TYPE:-"mysql"}
142 
143 # Spring datasource url, following :/? format, If you using mysql, you could use jdbc
144 
145 # string jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 as example
146 
147 SPRING_DATASOURCE_URL=${SPRING_DATASOURCE_URL:-"jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8"}
148 
149 # Spring datasource username
150 
151 SPRING_DATASOURCE_USERNAME=${SPRING_DATASOURCE_USERNAME:-"dolphin"}
152 
153 # Spring datasource password
154 
155 SPRING_DATASOURCE_PASSWORD=${SPRING_DATASOURCE_PASSWORD:-"dolphin"}
156 
157 # ---------------------------------------------------------
158 
159 # Registry Server
160 
161 # ---------------------------------------------------------
162 
163 # Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
164 
165 registryPluginName="zookeeper"
166 
167 # Registry Server address.
168 
169 registryServers="198.28.1.81:2181,198.28.1.82:2181,198.28.1.83:2181"
170 
171 # The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
172 
173 zkRoot="/dolphinscheduler"
174 
175 # ---------------------------------------------------------
176 
177 # Worker Task Server
178 
179 # ---------------------------------------------------------
180 
181 # Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
182 
183 taskPluginDir="lib/plugin/task"
184 
185 # resource storage type: HDFS, S3, NONE
186 
187 resourceStorageType="NONE"
188 
189 # resource store on HDFS/S3 path, resource file will store to this hdfs path, self configuration, please make sure the directory exists on hdfs and has read write permissions.
190 
191 "/dolphinscheduler" is recommended
192 
193 resourceUploadPath="/dolphinscheduler"
194 
195 # if resourceStorageType is HDFS，defaultFS write namenode address，HA, you need to put core-site.xml and hdfs-site.xml in the conf directory.
196 
197 # if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
198 
199 # Note，S3 be sure to create the root directory /dolphinscheduler
200 
201 defaultFS="hdfs://198.28.1.81:8020"
202 
203 # if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
204 
205 s3Endpoint="http://192.168.xx.xx:9010"
206 
207 s3AccessKey="xxxxxxxxxx"
208 
209 s3SecretKey="xxxxxxxxxx"
210 
211 # resourcemanager port, the default value is 8088 if not specified
212 
213 resourceManagerHttpAddressPort="8088"
214 
215 # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single node, keep this value empty
216 
217 yarnHaIps="192.168.xx.xx,192.168.xx.xx"
218 
219 # if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single node, you only need to replace 'yarnIp1' to actual re
220 
221 sourcemanager hostname
222 
223 singleYarnIp="yarnIp1"
224 
225 # who has permission to create directory under HDFS/S3 root path
226 
227 # Note: if kerberos is enabled, please config hdfsRootUser=
228 
229 hdfsRootUser="hdfs"
230 
231 # kerberos config
232 
233 # whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
234 
235 kerberosStartUp="false"
236 
237 # kdc krb5 config file path
238 
239 krb5ConfPath="$installPath/conf/krb5.conf"
240 
241 # keytab username,watch out the @ sign should followd by \\
242 
243 keytabUserName="hdfs"
244 
245 # username keytab path
246 
247 keytabPath="$installPath/conf/hdfs.keytab"
248 
249 # kerberos expire time, the unit is hour
250 
251 kerberosExpireTime="2"
252 
253 # use sudo or not
254 
255 sudoEnable="true"
256 
257 # worker tenant auto create
258 
259 workerTenantAutoCreate="false"

环境变量env/dolphinscheduler_env.sh 

 1 #
 2 
 3 # Licensed to the Apache Software Foundation (ASF) under one or more
 4 
 5 # contributor license agreements. See the NOTICE file distributed with
 6 
 7 # this work for additional information regarding copyright ownership.
 8 
 9 # The ASF licenses this file to You under the Apache License, Version 2.0
10 
11 # (the "License"); you may not use this file except in compliance with
12 
13 # the License. You may obtain a copy of the License at
14 
15 #
16 
17 # http://www.apache.org/licenses/LICENSE-2.0
18 
19 #
20 
21 # Unless required by applicable law or agreed to in writing, software
22 
23 # distributed under the License is distributed on an "AS IS" BASIS,
24 
25 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
26 
27 # See the License for the specific language governing permissions and
28 
29 # limitations under the License.
30 
31 #
32 
33 export HADOOP_HOME=/opt/soft/hadoop
34 
35 export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
36 
37 export SPARK_HOME1=/opt/soft/spark1
38 
39 export SPARK_HOME2=/opt/soft/spark2
40 
41 export PYTHON_HOME=/opt/soft/python
42 
43 export JAVA_HOME=/usr/java/jdk1.8.0_144
44 
45 export HIVE_HOME=/opt/soft/hive
46 
47 export FLINK_HOME=/opt/soft/flink
48 
49 export DATAX_HOME=/opt/soft/datax
50 
51 export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH

sh install.sh执行根目录install.sh安装部署

会出现启动成功的状态

访问页面

http://198.28.1.84:12345/dolphinscheduler

下次说下功能使用吧，谢谢

问题解决：

1.mysql开启外网访问，参考https://www.cnblogs.com/ningy1009/p/12806748.html