正在显示
24 个修改的文件
包含
198 行增加
和
689 行删除
@@ -7,23 +7,15 @@ ENV LANG=en_US.UTF-8 | @@ -7,23 +7,15 @@ ENV LANG=en_US.UTF-8 | ||
7 | #设置时区 | 7 | #设置时区 |
8 | ENV TZ=Asia/Shanghai | 8 | ENV TZ=Asia/Shanghai |
9 | 9 | ||
10 | +COPY . . | ||
10 | 11 | ||
11 | RUN apt-get update | 12 | RUN apt-get update |
12 | -#安装pip | 13 | +#安装pip3 |
13 | RUN apt-get install python3-pip -y | 14 | RUN apt-get install python3-pip -y |
14 | - | ||
15 | -#安装opencv所需依赖库 | ||
16 | -RUN apt-get install libgl1-mesa-glx -y | ||
17 | -RUN apt-get install libglib2.0-dev -y | ||
18 | - | ||
19 | #安装apache2 | 15 | #安装apache2 |
20 | -RUN apt-get install apache2 -y | ||
21 | -RUN apt-get install apache2-dev -y | ||
22 | - | ||
23 | - | 16 | +RUN apt-get install apache2 -y && apt-get install apache2-dev -y |
24 | #安装依赖 | 17 | #安装依赖 |
25 | -COPY . . | ||
26 | RUN pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ --trusted-host pypi.tuna.tsinghua.edu.cn | 18 | RUN pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ --trusted-host pypi.tuna.tsinghua.edu.cn |
27 | 19 | ||
20 | +RUN mod_wsgi-express install-module | ||
28 | 21 | ||
29 | -RUN mod_wsgi-express install-module |
BuildImage/requirements.txt
0 → 100644
1 | +flask==1.1.2 | ||
2 | +SQLAlchemy==1.3.17 | ||
3 | +Flask-SQLAlchemy==2.4.3 | ||
4 | +flask_cors==3.0.8 | ||
5 | +flasgger==0.9.5 | ||
6 | +psycopg2-binary==2.8.5 | ||
7 | +pyDes==2.0.1 | ||
8 | +mod_wsgi==4.8.0 | ||
9 | +opencv-python-headless==4.5.1.48 | ||
10 | +thrift==0.13.0 | ||
11 | +Authlib==0.13 | ||
12 | +kazoo==2.8.0 | ||
13 | +paramiko==2.8.0 | ||
14 | +requests==2.26.0 | ||
15 | +schedule==1.1.0 |
BuildImage/second-build/Dockerfile
0 → 100644
1 | +From osgeo/gdal:ubuntu-small-3.4.1 | ||
2 | +WORKDIR /root | ||
3 | +#设置编码 | ||
4 | +ENV LANG=en_US.UTF-8 | ||
5 | +#设置时区 | ||
6 | +ENV TZ=Asia/Shanghai | ||
7 | + | ||
8 | +#安装apache2 | ||
9 | +RUN apt-get update | ||
10 | +RUN apt-get install apache2 -y | ||
11 | + | ||
12 | +COPY --from=dci/dmapserver:build-dev /usr/local /usr/local | ||
13 | +COPY --from=dci/dmapserver:build-dev /usr/lib/apache2/modules /usr/lib/apache2/modules | ||
14 | + | ||
15 | + |
@@ -16,6 +16,10 @@ from app.util.component.StructuredPrint import StructurePrint | @@ -16,6 +16,10 @@ from app.util.component.StructuredPrint import StructurePrint | ||
16 | from app.util.component.PGUtil import PGUtil | 16 | from app.util.component.PGUtil import PGUtil |
17 | import os | 17 | import os |
18 | from app.modules.monitor.schedule import start_schedule | 18 | from app.modules.monitor.schedule import start_schedule |
19 | +import datetime | ||
20 | + | ||
21 | + | ||
22 | + | ||
19 | 23 | ||
20 | 24 | ||
21 | class JSONEncoder(_JSONEncoder): | 25 | class JSONEncoder(_JSONEncoder): |
@@ -76,11 +80,13 @@ def create_app(): | @@ -76,11 +80,13 @@ def create_app(): | ||
76 | db.create_all(app=app) | 80 | db.create_all(app=app) |
77 | 81 | ||
78 | # 日志 | 82 | # 日志 |
83 | + | ||
79 | logging.basicConfig(level=configure.log_level) | 84 | logging.basicConfig(level=configure.log_level) |
80 | log_file = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "logs", "log.txt") | 85 | log_file = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "logs", "log.txt") |
81 | handler = logging.FileHandler(log_file, encoding='UTF-8') # 设置日志字符集和存储路径名字 | 86 | handler = logging.FileHandler(log_file, encoding='UTF-8') # 设置日志字符集和存储路径名字 |
82 | logging_format = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s') | 87 | logging_format = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s') |
83 | handler.setFormatter(logging_format) | 88 | handler.setFormatter(logging_format) |
89 | + | ||
84 | app.logger.addHandler(handler) | 90 | app.logger.addHandler(handler) |
85 | 91 | ||
86 | # 配置使用鉴权组件,不写无法认证授权 | 92 | # 配置使用鉴权组件,不写无法认证授权 |
@@ -98,7 +98,7 @@ class DataManager(BlueprintApi): | @@ -98,7 +98,7 @@ class DataManager(BlueprintApi): | ||
98 | return database_info.Api().result | 98 | return database_info.Api().result |
99 | 99 | ||
100 | @staticmethod | 100 | @staticmethod |
101 | - @bp.route('/Detail', methods=["POST"]) | 101 | + @bp.route('/Detail', methods=["GET","POST"]) |
102 | @swag_from(database_detail.Api.api_doc) | 102 | @swag_from(database_detail.Api.api_doc) |
103 | def api_database_detail(): | 103 | def api_database_detail(): |
104 | """ | 104 | """ |
@@ -13,14 +13,20 @@ from app.util.component.PGUtil import PGUtil | @@ -13,14 +13,20 @@ from app.util.component.PGUtil import PGUtil | ||
13 | import configure | 13 | import configure |
14 | from osgeo.ogr import DataSource | 14 | from osgeo.ogr import DataSource |
15 | from flask import current_app | 15 | from flask import current_app |
16 | +from authlib.integrations.flask_oauth2 import current_token | ||
17 | + | ||
18 | + | ||
16 | class Api(ApiTemplate): | 19 | class Api(ApiTemplate): |
17 | api_name = "删除数据库" | 20 | api_name = "删除数据库" |
18 | def process(self): | 21 | def process(self): |
19 | re ={} | 22 | re ={} |
20 | va_ds = None | 23 | va_ds = None |
21 | try: | 24 | try: |
22 | - | 25 | + |
23 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() | 26 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() |
27 | + if database.creator != current_token.user.username: | ||
28 | + raise Exception("缺乏权限!") | ||
29 | + | ||
24 | if configure.VACUATE_DB_URI: | 30 | if configure.VACUATE_DB_URI: |
25 | va_ds: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) | 31 | va_ds: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) |
26 | else: | 32 | else: |
@@ -62,7 +68,7 @@ class Api(ApiTemplate): | @@ -62,7 +68,7 @@ class Api(ApiTemplate): | ||
62 | "parameters":[ | 68 | "parameters":[ |
63 | {"name": "guid", | 69 | {"name": "guid", |
64 | "in": "formData", | 70 | "in": "formData", |
65 | - "type": "string","description":"数据库guid","required": "true"}, | 71 | + "type": "string","description":"数据库guid","required": "true"} |
66 | 72 | ||
67 | ], | 73 | ], |
68 | "responses":{ | 74 | "responses":{ |
@@ -28,6 +28,9 @@ class Api(ApiTemplate): | @@ -28,6 +28,9 @@ class Api(ApiTemplate): | ||
28 | database = Database.query.filter_by(guid=guid) | 28 | database = Database.query.filter_by(guid=guid) |
29 | dbase:Database = database.one_or_none() | 29 | dbase:Database = database.one_or_none() |
30 | 30 | ||
31 | + if dbase.creator != "": | ||
32 | + raise Exception("缺乏权限!") | ||
33 | + | ||
31 | update_dict={} | 34 | update_dict={} |
32 | 35 | ||
33 | if not dbase: | 36 | if not dbase: |
@@ -18,7 +18,7 @@ from sqlalchemy.orm import Session | @@ -18,7 +18,7 @@ from sqlalchemy.orm import Session | ||
18 | import configure | 18 | import configure |
19 | import datetime | 19 | import datetime |
20 | import multiprocessing | 20 | import multiprocessing |
21 | -from ..util.EntryDataVacuate import EntryDataVacuate | 21 | +from .util.EntryDataVacuate import EntryDataVacuate |
22 | from app.util.component.TaskController import TaskController | 22 | from app.util.component.TaskController import TaskController |
23 | from app.util.component.TaskWriter import TaskWriter | 23 | from app.util.component.TaskWriter import TaskWriter |
24 | class Api(ApiTemplate): | 24 | class Api(ApiTemplate): |
@@ -212,6 +212,13 @@ class Api(ApiTemplate): | @@ -212,6 +212,13 @@ class Api(ApiTemplate): | ||
212 | "in": "formData", | 212 | "in": "formData", |
213 | "type": "string", | 213 | "type": "string", |
214 | "description": "目录guid"}, | 214 | "description": "目录guid"}, |
215 | + | ||
216 | + {"name": "vacuate", | ||
217 | + "in": "formData", | ||
218 | + "type": "string", | ||
219 | + "description": "是否抽稀", | ||
220 | + "enum":[1,0]}, | ||
221 | + | ||
215 | {"name": "check_meta_only", | 222 | {"name": "check_meta_only", |
216 | "in": "formData", | 223 | "in": "formData", |
217 | "type": "int", | 224 | "type": "int", |
@@ -15,7 +15,7 @@ import json | @@ -15,7 +15,7 @@ import json | ||
15 | from app.util.component.ApiTemplate import ApiTemplate | 15 | from app.util.component.ApiTemplate import ApiTemplate |
16 | from app.util.component.ZipUtil import ZipUtil | 16 | from app.util.component.ZipUtil import ZipUtil |
17 | from app.util.component.FileProcess import FileProcess | 17 | from app.util.component.FileProcess import FileProcess |
18 | - | 18 | +import platform |
19 | 19 | ||
20 | class Api(ApiTemplate): | 20 | class Api(ApiTemplate): |
21 | api_name = "获取meta" | 21 | api_name = "获取meta" |
@@ -53,7 +53,12 @@ class Api(ApiTemplate): | @@ -53,7 +53,12 @@ class Api(ApiTemplate): | ||
53 | 53 | ||
54 | else: | 54 | else: |
55 | # 保存文件 | 55 | # 保存文件 |
56 | - parent = os.path.dirname(os.path.realpath(__file__)) | 56 | + # 保存在项目外 |
57 | + if platform.system().lower() == 'windows': | ||
58 | + parent = "C:/" | ||
59 | + else: | ||
60 | + parent = "/tmp" | ||
61 | + | ||
57 | dir_path, store_file = FileProcess.save(parent) | 62 | dir_path, store_file = FileProcess.save(parent) |
58 | store_path = ZipUtil.unzip(store_file) | 63 | store_path = ZipUtil.unzip(store_file) |
59 | 64 |
@@ -18,7 +18,6 @@ import datetime | @@ -18,7 +18,6 @@ import datetime | ||
18 | class EntryDataVacuate: | 18 | class EntryDataVacuate: |
19 | 19 | ||
20 | def entry(self,parameter): | 20 | def entry(self,parameter): |
21 | - # meta:dict = parameter.get("meta") | ||
22 | 21 | ||
23 | # 初始化任务 | 22 | # 初始化任务 |
24 | this_task = ThisTask(parameter) | 23 | this_task = ThisTask(parameter) |
@@ -28,6 +27,9 @@ class EntryDataVacuate: | @@ -28,6 +27,9 @@ class EntryDataVacuate: | ||
28 | _data_path=None | 27 | _data_path=None |
29 | try: | 28 | try: |
30 | metas: list = parameter.get("meta") | 29 | metas: list = parameter.get("meta") |
30 | + | ||
31 | + vacuate = int(parameter.get("vacuate",1)) | ||
32 | + | ||
31 | # 总的入库是否成功 | 33 | # 总的入库是否成功 |
32 | is_success=True | 34 | is_success=True |
33 | 35 | ||
@@ -75,7 +77,10 @@ class EntryDataVacuate: | @@ -75,7 +77,10 @@ class EntryDataVacuate: | ||
75 | if is_success: | 77 | if is_success: |
76 | # 更新任务为成功任务 | 78 | # 更新任务为成功任务 |
77 | this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) | 79 | this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) |
78 | - this_task.commit() | 80 | + try: |
81 | + this_task.commit() | ||
82 | + except: | ||
83 | + raise Exception("可能编码出错!") | ||
79 | else: | 84 | else: |
80 | # 更新任务为失败任务 | 85 | # 更新任务为失败任务 |
81 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | 86 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) |
@@ -83,6 +88,7 @@ class EntryDataVacuate: | @@ -83,6 +88,7 @@ class EntryDataVacuate: | ||
83 | this_task.rollback() | 88 | this_task.rollback() |
84 | 89 | ||
85 | except Exception as e: | 90 | except Exception as e: |
91 | + StructurePrint().print("herehere") | ||
86 | this_task.write_process("{} 任务结束!".format(e.__str__())) | 92 | this_task.write_process("{} 任务结束!".format(e.__str__())) |
87 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | 93 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) |
88 | StructurePrint().print(e.__str__(),"ERROR") | 94 | StructurePrint().print(e.__str__(),"ERROR") |
@@ -98,7 +104,7 @@ class EntryDataVacuate: | @@ -98,7 +104,7 @@ class EntryDataVacuate: | ||
98 | dir_path = os.path.dirname(dir_path) | 104 | dir_path = os.path.dirname(dir_path) |
99 | i+=1 | 105 | i+=1 |
100 | if i<30: | 106 | if i<30: |
101 | - shutil.rmtree(dir_path,True) | 107 | + # shutil.rmtree(dir_path,True) |
102 | StructurePrint().print("删除文件成功!") | 108 | StructurePrint().print("删除文件成功!") |
103 | else: | 109 | else: |
104 | raise Exception("找不到文件!") | 110 | raise Exception("找不到文件!") |
@@ -156,15 +162,18 @@ class EntryDataVacuate: | @@ -156,15 +162,18 @@ class EntryDataVacuate: | ||
156 | 162 | ||
157 | def entry_one_layer(self,layer: Layer,this_task,meta): | 163 | def entry_one_layer(self,layer: Layer,this_task,meta): |
158 | 164 | ||
159 | - # this_task.pg_ds.StartTransaction() | 165 | + |
160 | new_layer_name = None | 166 | new_layer_name = None |
161 | - # vacuate_process= None | 167 | + vacuate_process= None |
168 | + vacuate = int(this_task.parameter.get("vacuate", 1)) | ||
162 | success = True | 169 | success = True |
163 | table_guid = uuid.uuid1().__str__() | 170 | table_guid = uuid.uuid1().__str__() |
164 | try: | 171 | try: |
165 | # 图层设置 | 172 | # 图层设置 |
166 | parameter = this_task.parameter | 173 | parameter = this_task.parameter |
167 | 174 | ||
175 | + | ||
176 | + | ||
168 | overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" | 177 | overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" |
169 | geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" | 178 | geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" |
170 | fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" | 179 | fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" |
@@ -201,9 +210,8 @@ class EntryDataVacuate: | @@ -201,9 +210,8 @@ class EntryDataVacuate: | ||
201 | pg_layer.CreateFields(schema) | 210 | pg_layer.CreateFields(schema) |
202 | 211 | ||
203 | #创建抽稀过程 | 212 | #创建抽稀过程 |
204 | - # vacuate_process = VacuateProcess(layer,table_guid,options) | ||
205 | - | ||
206 | - | 213 | + if vacuate: |
214 | + vacuate_process = VacuateProcess(layer,table_guid,options) | ||
207 | 215 | ||
208 | count =0 | 216 | count =0 |
209 | 217 | ||
@@ -231,18 +239,18 @@ class EntryDataVacuate: | @@ -231,18 +239,18 @@ class EntryDataVacuate: | ||
231 | pg_layer.CreateFeature(out_feature) | 239 | pg_layer.CreateFeature(out_feature) |
232 | 240 | ||
233 | #插入抽稀图层 | 241 | #插入抽稀图层 |
234 | - # if out_geom is not None: | ||
235 | - # vacuate_process.vacuate(out_geom) | 242 | + if out_geom is not None and vacuate: |
243 | + vacuate_process.vacuate(out_geom) | ||
236 | 244 | ||
237 | # 注册图层信息 | 245 | # 注册图层信息 |
238 | # 是否抽吸过 | 246 | # 是否抽吸过 |
239 | - # is_vacuate = 1 if vacuate_process.max_level>0 else 0 | ||
240 | - is_vacuate = 0 | 247 | + is_vacuate = 1 if vacuate_process.max_level>0 else 0 |
241 | 248 | ||
242 | this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | 249 | this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) |
243 | 250 | ||
244 | # 注册抽稀表 | 251 | # 注册抽稀表 |
245 | - # this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | 252 | + if vacuate: |
253 | + this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | ||
246 | 254 | ||
247 | this_task.write_process("{}图层入库成功。".format(new_layer_name)) | 255 | this_task.write_process("{}图层入库成功。".format(new_layer_name)) |
248 | 256 | ||
@@ -252,11 +260,13 @@ class EntryDataVacuate: | @@ -252,11 +260,13 @@ class EntryDataVacuate: | ||
252 | StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") | 260 | StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") |
253 | print(traceback.format_exc()) | 261 | print(traceback.format_exc()) |
254 | # 抽稀回滚 | 262 | # 抽稀回滚 |
255 | - # vacuate_process.rollback() | 263 | + if vacuate: |
264 | + vacuate_process.rollback() | ||
256 | success =False | 265 | success =False |
257 | 266 | ||
258 | finally: | 267 | finally: |
259 | - # vacuate_process.end() | 268 | + if vacuate: |
269 | + vacuate_process.end() | ||
260 | pass | 270 | pass |
261 | return success,new_layer_name | 271 | return success,new_layer_name |
262 | 272 | ||
@@ -459,12 +469,6 @@ class VacuateProcess: | @@ -459,12 +469,6 @@ class VacuateProcess: | ||
459 | # 固有疏密程度 | 469 | # 固有疏密程度 |
460 | original_density=8 | 470 | original_density=8 |
461 | 471 | ||
462 | - | ||
463 | - # 额外一层 | ||
464 | - # self.this_gridsize.append(0.000075) | ||
465 | - # self.max_level += 1 | ||
466 | - ###### | ||
467 | - | ||
468 | if extent[0]>180: | 472 | if extent[0]>180: |
469 | self.t_grid_size=self.project_gridsize | 473 | self.t_grid_size=self.project_gridsize |
470 | else: | 474 | else: |
@@ -19,6 +19,14 @@ class Api(ApiTemplate): | @@ -19,6 +19,14 @@ class Api(ApiTemplate): | ||
19 | 19 | ||
20 | column_guid = self.para.get("column_guid") | 20 | column_guid = self.para.get("column_guid") |
21 | update_dict = {} | 21 | update_dict = {} |
22 | + | ||
23 | + column:Columns = Columns.query.filter_by(guid = column_guid).one_or_none() | ||
24 | + if not column: | ||
25 | + raise Exception("数据不存在!") | ||
26 | + | ||
27 | + if column.relate_table.relate_database.creator != "": | ||
28 | + raise Exception("缺乏权限!") | ||
29 | + | ||
22 | try: | 30 | try: |
23 | if self.para.get("column_alias"): | 31 | if self.para.get("column_alias"): |
24 | update_dict["alias"] = self.para.get("column_alias") | 32 | update_dict["alias"] = self.para.get("column_alias") |
@@ -47,7 +47,10 @@ class Api(ApiTemplate): | @@ -47,7 +47,10 @@ class Api(ApiTemplate): | ||
47 | res["msg"]= "数据库不存在!" | 47 | res["msg"]= "数据库不存在!" |
48 | return res | 48 | return res |
49 | 49 | ||
50 | - | 50 | + if database.creator != "": |
51 | + raise Exception("缺乏权限!") | ||
52 | + | ||
53 | + | ||
51 | pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri)) | 54 | pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri)) |
52 | 55 | ||
53 | if configure.VACUATE_DB_URI: | 56 | if configure.VACUATE_DB_URI: |
@@ -34,6 +34,8 @@ class Api(ApiTemplate): | @@ -34,6 +34,8 @@ class Api(ApiTemplate): | ||
34 | res["msg"]= "数据不存在!" | 34 | res["msg"]= "数据不存在!" |
35 | return res | 35 | return res |
36 | 36 | ||
37 | + if table.relate_database.creator != "": | ||
38 | + raise Exception("缺乏权限!") | ||
37 | 39 | ||
38 | if self.para.__contains__("catalog_guid"): | 40 | if self.para.__contains__("catalog_guid"): |
39 | if catalog_guid is None: | 41 | if catalog_guid is None: |
@@ -21,9 +21,10 @@ class Api(ApiTemplate): | @@ -21,9 +21,10 @@ class Api(ApiTemplate): | ||
21 | raise Exception("数据不存在!") | 21 | raise Exception("数据不存在!") |
22 | pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 22 | pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
23 | layer:Layer = pg_ds.GetLayerByName(table.name) | 23 | layer:Layer = pg_ds.GetLayerByName(table.name) |
24 | - append_dict ={"srid":None,"sr_wkt":None,"sr_proj4":None,"exist":1} | 24 | + append_dict ={"srid":None,"sr_wkt":None,"sr_proj4":None,"exist":1,"schema":"public","geomfield":layer.GetGeometryColumn()} |
25 | if layer: | 25 | if layer: |
26 | sr:SpatialReference = layer.GetSpatialRef() | 26 | sr:SpatialReference = layer.GetSpatialRef() |
27 | + | ||
27 | if sr: | 28 | if sr: |
28 | append_dict["srid"] = sr.GetAuthorityCode(None) | 29 | append_dict["srid"] = sr.GetAuthorityCode(None) |
29 | append_dict["sr_wkt"] = sr.ExportToWkt() | 30 | append_dict["sr_wkt"] = sr.ExportToWkt() |
@@ -32,6 +32,9 @@ class Api(ApiTemplate): | @@ -32,6 +32,9 @@ class Api(ApiTemplate): | ||
32 | database_guid = self.para.get("database_guid") | 32 | database_guid = self.para.get("database_guid") |
33 | database = Database.query.filter_by(guid=database_guid).one_or_none() | 33 | database = Database.query.filter_by(guid=database_guid).one_or_none() |
34 | 34 | ||
35 | + if database.creator != "": | ||
36 | + raise Exception("缺乏权限!") | ||
37 | + | ||
35 | if not database: | 38 | if not database: |
36 | raise Exception("数据库不存在!") | 39 | raise Exception("数据库不存在!") |
37 | # 初始化task | 40 | # 初始化task |
@@ -42,6 +42,9 @@ class Api(ApiTemplate): | @@ -42,6 +42,9 @@ class Api(ApiTemplate): | ||
42 | if not table: | 42 | if not table: |
43 | raise Exception("数据不存在!") | 43 | raise Exception("数据不存在!") |
44 | 44 | ||
45 | + if table.relate_database.creator != "": | ||
46 | + raise Exception("缺乏权限!") | ||
47 | + | ||
45 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 48 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
46 | layer = pg_ds.GetLayerByName(table.name) | 49 | layer = pg_ds.GetLayerByName(table.name) |
47 | # 判断图层是否存在 | 50 | # 判断图层是否存在 |
@@ -73,7 +76,7 @@ class Api(ApiTemplate): | @@ -73,7 +76,7 @@ class Api(ApiTemplate): | ||
73 | 76 | ||
74 | 77 | ||
75 | task = Task(guid=task_guid, | 78 | task = Task(guid=task_guid, |
76 | - name="{}矢量金字塔构建".format(table.name), | 79 | + name="{}构建矢量金字塔".format(table.name), |
77 | table_guid=table_guid, | 80 | table_guid=table_guid, |
78 | create_time=datetime.datetime.now(), | 81 | create_time=datetime.datetime.now(), |
79 | state=0, | 82 | state=0, |
@@ -40,6 +40,9 @@ class Api(ApiTemplate): | @@ -40,6 +40,9 @@ class Api(ApiTemplate): | ||
40 | if not table: | 40 | if not table: |
41 | raise Exception("数据不存在!") | 41 | raise Exception("数据不存在!") |
42 | 42 | ||
43 | + if table.relate_database.creator != "": | ||
44 | + raise Exception("缺乏权限!") | ||
45 | + | ||
43 | # 判断图层是否存在 | 46 | # 判断图层是否存在 |
44 | 47 | ||
45 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 48 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
@@ -73,7 +76,7 @@ class Api(ApiTemplate): | @@ -73,7 +76,7 @@ class Api(ApiTemplate): | ||
73 | 76 | ||
74 | 77 | ||
75 | task = Task(guid=task_guid, | 78 | task = Task(guid=task_guid, |
76 | - name="{}矢量金字塔构建,网格大小:{}".format(table.name,self.para.get("grids")), | 79 | + name="{}构建矢量金字塔,网格大小:{}".format(table.name,self.para.get("grids")), |
77 | table_guid=table_guid, | 80 | table_guid=table_guid, |
78 | create_time=datetime.datetime.now(), | 81 | create_time=datetime.datetime.now(), |
79 | state=0, | 82 | state=0, |
1 | - | ||
2 | -from osgeo.ogr import * | ||
3 | -from osgeo import ogr | ||
4 | -from osgeo import gdal | ||
5 | -import os | ||
6 | -import uuid | ||
7 | -import shutil | ||
8 | -import time | ||
9 | -from app.modules.data.models import * | ||
10 | -from app.util.component.PGUtil import PGUtil | ||
11 | -from app.util.component.StructuredPrint import StructurePrint | ||
12 | -from sqlalchemy.orm import Session | ||
13 | -import configure | ||
14 | -import math | ||
15 | -from functools import lru_cache | ||
16 | -import traceback | ||
17 | -import copy | ||
18 | -from app.util.component.GeometryAdapter import GeometryAdapter | ||
19 | -from app.util.component.VacuateConf import VacuateConf | ||
20 | -import datetime | ||
21 | -class EntryDataVacuate: | ||
22 | - | ||
23 | - def entry(self,parameter): | ||
24 | - # meta:dict = parameter.get("meta") | ||
25 | - | ||
26 | - # 初始化任务 | ||
27 | - this_task = ThisTask(parameter) | ||
28 | - this_task.write_process("入库任务初始化...") | ||
29 | - | ||
30 | - # 数据路径,用作删除 | ||
31 | - _data_path=None | ||
32 | - try: | ||
33 | - metas: list = parameter.get("meta") | ||
34 | - # 总的入库是否成功 | ||
35 | - is_success=True | ||
36 | - | ||
37 | - this_task.update({"process": "入库中"}) | ||
38 | - | ||
39 | - # 开始入库事务 | ||
40 | - this_task.start() | ||
41 | - | ||
42 | - # 多个文件依次入库 | ||
43 | - for meta in metas: | ||
44 | - #设置编码 | ||
45 | - encoding = parameter.get("encoding") | ||
46 | - if encoding: | ||
47 | - gdal.SetConfigOption("SHAPE_ENCODING",encoding) | ||
48 | - else: | ||
49 | - gdal.SetConfigOption("SHAPE_ENCODING", "GBK") | ||
50 | - | ||
51 | - #如果包含cpg文件,优先使用cpg文件中声明的编码 | ||
52 | - encoding_cpg = meta.get("encoding") | ||
53 | - if encoding_cpg: | ||
54 | - gdal.SetConfigOption("SHAPE_ENCODING", encoding_cpg) | ||
55 | - | ||
56 | - data_path = meta.get("data_path") | ||
57 | - | ||
58 | - #设定删除路径 | ||
59 | - if not _data_path: | ||
60 | - _data_path=data_path | ||
61 | - | ||
62 | - | ||
63 | - if not data_path: | ||
64 | - raise Exception("数据错误!") | ||
65 | - # 分为shp和gdb 2种录入形式 | ||
66 | - | ||
67 | - if data_path.endswith("shp"): | ||
68 | - is_success_one,new_layer_name = self.entry_shp(data_path,this_task,meta) | ||
69 | - else: | ||
70 | - is_success_one,new_layer_names = self.entry_gdb(data_path,this_task,meta) | ||
71 | - | ||
72 | - #如果其中一个失败,总的入库就失败 | ||
73 | - if not is_success_one: | ||
74 | - is_success=False | ||
75 | - | ||
76 | - this_task.write_process("数据入库结束。") | ||
77 | - | ||
78 | - if is_success: | ||
79 | - # 更新任务为成功任务 | ||
80 | - this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) | ||
81 | - this_task.commit() | ||
82 | - else: | ||
83 | - # 更新任务为失败任务 | ||
84 | - this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | ||
85 | - # rollback | ||
86 | - this_task.rollback() | ||
87 | - | ||
88 | - except Exception as e: | ||
89 | - this_task.write_process(e.__str__()) | ||
90 | - this_task.write_process("任务中止!") | ||
91 | - | ||
92 | - this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | ||
93 | - StructurePrint().print(e.__str__(),"ERROR") | ||
94 | - # rollback | ||
95 | - this_task.rollback() | ||
96 | - finally: | ||
97 | - this_task.end() | ||
98 | - try: | ||
99 | - file_tmp_path = os.path.join(_data_path.split("file_tmp")[0],"file_tmp") | ||
100 | - dir_path = os.path.dirname(_data_path) | ||
101 | - i=0 | ||
102 | - while not os.path.dirname(dir_path).__eq__(file_tmp_path) and i<30: | ||
103 | - dir_path = os.path.dirname(dir_path) | ||
104 | - i+=1 | ||
105 | - if i<30: | ||
106 | - shutil.rmtree(dir_path,True) | ||
107 | - StructurePrint().print("删除文件成功!") | ||
108 | - else: | ||
109 | - raise Exception("找不到文件!") | ||
110 | - | ||
111 | - except Exception as e: | ||
112 | - StructurePrint().print(e.__str__(), "ERROR") | ||
113 | - StructurePrint().print("删除文件失败!","ERROR") | ||
114 | - | ||
115 | - | ||
116 | - def entry_shp(self,data_path,this_task,meta): | ||
117 | - ''' | ||
118 | - 录入shp | ||
119 | - :param data_path: | ||
120 | - :return: | ||
121 | - ''' | ||
122 | - | ||
123 | - driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | ||
124 | - ds: DataSource = driver.Open(data_path, 1) | ||
125 | - if not ds: | ||
126 | - raise Exception("打开数据失败!") | ||
127 | - layer: Layer = ds.GetLayer(0) | ||
128 | - | ||
129 | - is_success_one, new_layer_name =self.entry_one_layer(layer, this_task,meta) | ||
130 | - ds.Destroy() | ||
131 | - return is_success_one, new_layer_name | ||
132 | - | ||
133 | - def entry_gdb(self,data_path,this_task,meta): | ||
134 | - ''' | ||
135 | - 录入gdb | ||
136 | - :param data_path: | ||
137 | - :return: | ||
138 | - ''' | ||
139 | - | ||
140 | - is_successes = [] | ||
141 | - new_layer_names=[] | ||
142 | - driver: Driver = ogr.GetDriverByName("OpenFileGDB") | ||
143 | - ds: DataSource = driver.Open(data_path, 0) | ||
144 | - if not ds: | ||
145 | - raise Exception("打开数据失败!") | ||
146 | - | ||
147 | - | ||
148 | - for i in range(ds.GetLayerCount()): | ||
149 | - | ||
150 | - layer: Layer = ds.GetLayer(i) | ||
151 | - if layer.GetName().lower() not in meta.get("layer").keys(): | ||
152 | - continue | ||
153 | - is_success, new_layer_name = self.entry_one_layer(layer,this_task,meta) | ||
154 | - new_layer_names.append(new_layer_name) | ||
155 | - is_successes.append(is_success) | ||
156 | - ds.Destroy() | ||
157 | - if is_successes.__contains__(False): | ||
158 | - return False,new_layer_names | ||
159 | - else: | ||
160 | - return True,new_layer_names | ||
161 | - | ||
162 | - def entry_one_layer(self,layer: Layer,this_task,meta): | ||
163 | - | ||
164 | - # this_task.pg_ds.StartTransaction() | ||
165 | - new_layer_name = None | ||
166 | - # vacuate_process= None | ||
167 | - success = True | ||
168 | - table_guid = uuid.uuid1().__str__() | ||
169 | - try: | ||
170 | - # 图层设置 | ||
171 | - parameter = this_task.parameter | ||
172 | - | ||
173 | - overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" | ||
174 | - geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" | ||
175 | - fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" | ||
176 | - options = ["OVERWRITE={}".format(overwrite), "FID={}".format(fid), "GEOMETRY_NAME={}".format(geom_name),"PRECISION=NO"] | ||
177 | - | ||
178 | - | ||
179 | - # 将线/面转多线多面 | ||
180 | - geom_type = GeometryAdapter.change_geom_type(layer.GetGeomType()) | ||
181 | - | ||
182 | - # 更改图层名 | ||
183 | - change_name = False | ||
184 | - origin_name = layer.GetName().lower() | ||
185 | - | ||
186 | - # 新图层名 | ||
187 | - new_layer_name: str = meta.get("layer").get(origin_name) | ||
188 | - origin_name = new_layer_name | ||
189 | - no = 1 | ||
190 | - while overwrite.__eq__("no") and this_task.pg_ds.GetLayerByName(new_layer_name) : | ||
191 | - change_name=True | ||
192 | - new_layer_name = origin_name+"_{}".format(no) | ||
193 | - no+=1 | ||
194 | - | ||
195 | - if change_name: | ||
196 | - this_task.write_process("{}图层已存在,更名为{}入库".format(origin_name, new_layer_name)) | ||
197 | - | ||
198 | - | ||
199 | - this_task.write_process("{}图层正在入库...".format(new_layer_name)) | ||
200 | - | ||
201 | - pg_layer: Layer = this_task.pg_ds.CreateLayer(new_layer_name, layer.GetSpatialRef(), geom_type, options) | ||
202 | - | ||
203 | - # 复制原图层的属性 | ||
204 | - # 去掉fid的属性 | ||
205 | - schema = [sche for sche in layer.schema if not sche.name.__eq__(fid)] | ||
206 | - pg_layer.CreateFields(schema) | ||
207 | - | ||
208 | - #创建抽稀过程 | ||
209 | - # vacuate_process = VacuateProcess(layer,table_guid,options) | ||
210 | - | ||
211 | - | ||
212 | - | ||
213 | - count =0 | ||
214 | - | ||
215 | - for feature in layer: | ||
216 | - count+=1 | ||
217 | - if count%10000==0: | ||
218 | - StructurePrint().print("{}图层已入库{}个对象".format(new_layer_name,count)) | ||
219 | - # print(time.time()-this_time) | ||
220 | - #this_time=time.time() | ||
221 | - geo :Geometry = feature.GetGeometryRef() | ||
222 | - # 如果是空对象不录入 | ||
223 | - if geo is not None: | ||
224 | - if geo.IsEmpty(): | ||
225 | - this_task.write_process("FID:{}要素的空间字段为空,跳过该要素!".format(feature.GetFID())) | ||
226 | - StructurePrint().print("FID:{}要素的空间字段为空,跳过该要素!".format(feature.GetFID()),"WARN") | ||
227 | - continue | ||
228 | - | ||
229 | - out_feature: Feature = copy.copy(feature) | ||
230 | - out_geom = None | ||
231 | - if geo is not None: | ||
232 | - out_geom:Geometry = GeometryAdapter.change_geom(geo, geom_type) | ||
233 | - out_feature.SetGeometry(out_geom) | ||
234 | - # 出现fid为0经常有问题 | ||
235 | - out_feature.SetFID(out_feature.GetFID() + 1) | ||
236 | - pg_layer.CreateFeature(out_feature) | ||
237 | - | ||
238 | - #插入抽稀图层 | ||
239 | - # if out_geom is not None: | ||
240 | - # vacuate_process.vacuate(out_geom) | ||
241 | - | ||
242 | - # 注册图层信息 | ||
243 | - # 是否抽吸过 | ||
244 | - # is_vacuate = 1 if vacuate_process.max_level>0 else 0 | ||
245 | - is_vacuate = 0 | ||
246 | - | ||
247 | - this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | ||
248 | - | ||
249 | - # 注册抽稀表 | ||
250 | - # this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | ||
251 | - | ||
252 | - this_task.write_process("{}图层入库成功。".format(new_layer_name)) | ||
253 | - | ||
254 | - except Exception as e: | ||
255 | - | ||
256 | - this_task.write_process("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__())) | ||
257 | - StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") | ||
258 | - print(traceback.format_exc()) | ||
259 | - # 抽稀回滚 | ||
260 | - # vacuate_process.rollback() | ||
261 | - success =False | ||
262 | - | ||
263 | - finally: | ||
264 | - # vacuate_process.end() | ||
265 | - pass | ||
266 | - return success,new_layer_name | ||
267 | - | ||
268 | - | ||
269 | - | ||
270 | -class ThisTask: | ||
271 | - | ||
272 | - def __init__(self, parameter): | ||
273 | - try: | ||
274 | - # 该任务事务的连接 | ||
275 | - self.sys_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
276 | - # 专门的写过程的连接 | ||
277 | - self.process_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
278 | - | ||
279 | - except Exception as e: | ||
280 | - raise Exception("打开数据库失败!") | ||
281 | - self.parameter = parameter | ||
282 | - | ||
283 | - self.task = self.process_session.query(Task).filter_by(guid=parameter.get("task_guid")) | ||
284 | - | ||
285 | - self.database = self.sys_session.query(Database).filter_by( | ||
286 | - guid=parameter.get("database_guid")).one_or_none() | ||
287 | - | ||
288 | - self.catalog_guid = parameter.get("catalog_guid") | ||
289 | - | ||
290 | - self.pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(self.database.sqlalchemy_uri)) | ||
291 | - | ||
292 | - | ||
293 | - def start(self): | ||
294 | - self.pg_ds.StartTransaction() | ||
295 | - | ||
296 | - def update(self, update_dict): | ||
297 | - self.task.update(update_dict) | ||
298 | - self.process_session.commit() | ||
299 | - | ||
300 | - def write_process(self, message): | ||
301 | - message = "{} {}".format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), message) | ||
302 | - task_process_guid = uuid.uuid1().__str__() | ||
303 | - task_process = Process(guid=task_process_guid, message=message, time=datetime.datetime.now(), | ||
304 | - task_guid=self.parameter.get("task_guid")) | ||
305 | - self.process_session.add(task_process) | ||
306 | - self.process_session.commit() | ||
307 | - | ||
308 | - def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate,table_guid): | ||
309 | - ''' | ||
310 | - 注册表 | ||
311 | - :param layer: 图层 | ||
312 | - :param new_layer_name: 图层名 | ||
313 | - :return: 表名 | ||
314 | - ''' | ||
315 | - | ||
316 | - this_time = datetime.datetime.now() | ||
317 | - | ||
318 | - ext = layer.GetExtent() | ||
319 | - if ext[0] < 360: | ||
320 | - ext = [round(e, 6) for e in ext] | ||
321 | - else: | ||
322 | - ext = [round(e, 2) for e in ext] | ||
323 | - | ||
324 | - geom_type = GeometryAdapter.get_geometry_type(layer) | ||
325 | - | ||
326 | - extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) | ||
327 | - | ||
328 | - table = Table(guid=table_guid, | ||
329 | - database_guid=self.database.guid, | ||
330 | - creator=creator, | ||
331 | - name=new_layer_name, create_time=this_time, update_time=this_time, | ||
332 | - catalog_guid=self.catalog_guid, table_type=GeometryAdapter.get_table_type(geom_type), | ||
333 | - extent=extent, | ||
334 | - feature_count=layer.GetFeatureCount(), | ||
335 | - is_vacuate=is_vacuate | ||
336 | - ) | ||
337 | - # 删除遗留业务数据 | ||
338 | - try: | ||
339 | - history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all() | ||
340 | - except: | ||
341 | - self.sys_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
342 | - history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all() | ||
343 | - | ||
344 | - if history_table: | ||
345 | - for ht in history_table: | ||
346 | - self.sys_session.delete(ht) | ||
347 | - self.sys_session.add(table) | ||
348 | - | ||
349 | - feature_defn: FeatureDefn = layer.GetLayerDefn() | ||
350 | - | ||
351 | - for i in range(feature_defn.GetFieldCount()): | ||
352 | - field_defn: FieldDefn = feature_defn.GetFieldDefn(i) | ||
353 | - field_name = field_defn.GetName().lower() | ||
354 | - field_alias = field_name if field_defn.GetAlternativeName() is None or field_defn.GetAlternativeName().__eq__( | ||
355 | - "") else field_defn.GetAlternativeName() | ||
356 | - column = Columns(guid=uuid.uuid1().__str__(), table_guid=table_guid, | ||
357 | - name=field_name, alias=field_alias, create_time=this_time, update_time=this_time) | ||
358 | - self.sys_session.add(column) | ||
359 | - return table_guid | ||
360 | - | ||
361 | - | ||
362 | - def register_table_vacuate(self,table_guid,vacuate_layers:dict): | ||
363 | - | ||
364 | - # 抽稀表有固定的命名规则 | ||
365 | - for level,layer in vacuate_layers.items(): | ||
366 | - pixel_distance_str:str=layer.GetName().split("_")[-1] | ||
367 | - lev = layer.GetName().split("_")[-2] | ||
368 | - | ||
369 | - if pixel_distance_str.startswith("0"): | ||
370 | - pixel_distance_str="0.{}".format(pixel_distance_str) | ||
371 | - | ||
372 | - pixel_distance = float(pixel_distance_str) | ||
373 | - | ||
374 | - table_vacuate = TableVacuate(guid=uuid.uuid1().__str__(), | ||
375 | - table_guid=table_guid, | ||
376 | - level=int(lev), | ||
377 | - name=layer.GetName(), | ||
378 | - pixel_distance=pixel_distance) | ||
379 | - self.sys_session.add(table_vacuate) | ||
380 | - | ||
381 | - | ||
382 | - def commit(self): | ||
383 | - if self.sys_session: | ||
384 | - self.sys_session.commit() | ||
385 | - if self.pg_ds: | ||
386 | - self.pg_ds.CommitTransaction() | ||
387 | - if self.process_session: | ||
388 | - self.process_session.commit() | ||
389 | - | ||
390 | - | ||
391 | - def end(self): | ||
392 | - if self.sys_session: | ||
393 | - self.sys_session.close() | ||
394 | - if self.pg_ds: | ||
395 | - self.pg_ds.Destroy() | ||
396 | - if self.process_session: | ||
397 | - self.process_session.close() | ||
398 | - | ||
399 | - def rollback(self): | ||
400 | - if self.sys_session: | ||
401 | - self.sys_session.rollback() | ||
402 | - if self.pg_ds: | ||
403 | - self.pg_ds.RollbackTransaction() | ||
404 | - | ||
405 | - | ||
406 | - | ||
407 | -class VacuateProcess: | ||
408 | - | ||
409 | - max_level=0 | ||
410 | - fill_dict={} | ||
411 | - vacuate_layers={} | ||
412 | - vacuate_layers_gridsize={} | ||
413 | - pg_ds_dict = {} | ||
414 | - # 图层要素大于5W才抽稀 | ||
415 | - least_vacuate_count = VacuateConf.least_vacuate_count | ||
416 | - | ||
417 | - extent=[] | ||
418 | - is_spatial=False | ||
419 | - | ||
420 | - lonlat_gridsize = VacuateConf.lonlat_gridsize | ||
421 | - project_gridsize = VacuateConf.project_gridsize | ||
422 | - | ||
423 | - # 该抽稀过程使用的grid_size | ||
424 | - t_grid_size = [] | ||
425 | - | ||
426 | - # 该抽稀过程的抽稀网格 | ||
427 | - this_gridsize=[] | ||
428 | - | ||
429 | - | ||
430 | - def __init__(self,layer:Layer,table_guid, options,sqlalchemy_uri): | ||
431 | - | ||
432 | - #是空间图层才初始化 | ||
433 | - if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: | ||
434 | - | ||
435 | - self.is_spatial=True | ||
436 | - | ||
437 | - # 判断需要抽稀多少级 | ||
438 | - | ||
439 | - lc = layer.GetFeatureCount() | ||
440 | - extent = layer.GetExtent() | ||
441 | - self.extent=extent | ||
442 | - | ||
443 | - #判断疏密程度 | ||
444 | - p_x = (extent[1]-extent[0])/10.0 | ||
445 | - p_y = (extent[3] - extent[2]) / 10.0 | ||
446 | - fill_precent=0 | ||
447 | - StructurePrint().print("判断疏密") | ||
448 | - for ix in range(10): | ||
449 | - for iy in range(10): | ||
450 | - grid_extent = [extent[0]+ix*p_x,extent[0]+ix*p_x+p_x,extent[2]+iy*p_y,extent[2]+iy*p_y+p_y] | ||
451 | - poly = GeometryAdapter.envelop_2_polygon(grid_extent) | ||
452 | - | ||
453 | - layer.SetSpatialFilter(None) | ||
454 | - layer.SetSpatialFilter(poly) | ||
455 | - layer.ResetReading() | ||
456 | - if layer.GetNextFeature(): | ||
457 | - fill_precent += 1 | ||
458 | - | ||
459 | - print(fill_precent) | ||
460 | - StructurePrint().print("判断疏密结束") | ||
461 | - | ||
462 | - layer.SetSpatialFilter(None) | ||
463 | - layer.ResetReading() | ||
464 | - # 固有疏密程度 | ||
465 | - original_density=8 | ||
466 | - | ||
467 | - | ||
468 | - # 额外一层 | ||
469 | - # self.this_gridsize.append(0.000075) | ||
470 | - # self.max_level += 1 | ||
471 | - ###### | ||
472 | - | ||
473 | - if extent[0]>180: | ||
474 | - self.t_grid_size=self.project_gridsize | ||
475 | - else: | ||
476 | - self.t_grid_size = self.lonlat_gridsize | ||
477 | - | ||
478 | - for grid_size in self.t_grid_size: | ||
479 | - # 最少抽稀个数 | ||
480 | - if lc > self.least_vacuate_count: | ||
481 | - # 网格数至少大于 | ||
482 | - if ((extent[1] - extent[0]) * (extent[3] - extent[2])) / (grid_size**2)>self.least_vacuate_count: | ||
483 | - # 要素数量大于网格数量 | ||
484 | - # 要考虑图层的疏密程度,original_density*(100.0/fill_precent) 为疏密指数 | ||
485 | - if lc * original_density * (100.0/fill_precent)>((extent[1] - extent[0])*(extent[3] - extent[2]))/(grid_size**2) : | ||
486 | - print(grid_size) | ||
487 | - self.this_gridsize.append(grid_size) | ||
488 | - self.max_level += 1 | ||
489 | - | ||
490 | - | ||
491 | - | ||
492 | - # 创建抽稀ds | ||
493 | - for l in range(self.max_level): | ||
494 | - # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | ||
495 | - if configure.VACUATE_DB_URI: | ||
496 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) | ||
497 | - else: | ||
498 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | ||
499 | - pg_ds_l.StartTransaction() | ||
500 | - self.pg_ds_dict[l] = pg_ds_l | ||
501 | - | ||
502 | - # 生成抽稀图层 | ||
503 | - options = options[1:] | ||
504 | - options.append("OVERWRITE=yes") | ||
505 | - options.append("LAUNDER=no") | ||
506 | - | ||
507 | - schema = layer.schema | ||
508 | - # 增加统计字段 | ||
509 | - schema.append(ogr.FieldDefn("_dcigrid_count_", ogr.OFTInteger)) | ||
510 | - schema.append(ogr.FieldDefn("_dcigrid_name_", ogr.OFTString)) | ||
511 | - | ||
512 | - for l in range(self.max_level): | ||
513 | - this_grid_len = self.this_gridsize[l] | ||
514 | - | ||
515 | - self.vacuate_layers_gridsize[l] = this_grid_len | ||
516 | - | ||
517 | - pg = self.pg_ds_dict[l] | ||
518 | - | ||
519 | - grid_name = str(this_grid_len) | ||
520 | - if this_grid_len<1: | ||
521 | - grid_name = str(this_grid_len).split(".")[-1] | ||
522 | - if this_grid_len.__eq__(0.00008): | ||
523 | - grid_name = "00008" | ||
524 | - | ||
525 | - # 抽稀图层是点面混合的 | ||
526 | - # 抽稀表有固定的命名规则 | ||
527 | - # 抽稀表一定要覆盖 | ||
528 | - | ||
529 | - | ||
530 | - print("{}:{}".format(self.t_grid_size.index(this_grid_len),this_grid_len)) | ||
531 | - | ||
532 | - | ||
533 | - v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name) | ||
534 | - vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | ||
535 | - # 抽稀表需要属性 | ||
536 | - vl.CreateFields(schema) | ||
537 | - self.vacuate_layers[l] = vl | ||
538 | - | ||
539 | - else: | ||
540 | - pass | ||
541 | - | ||
542 | - | ||
543 | - def vacuate(self,g,feature): | ||
544 | - | ||
545 | - if self.is_spatial: | ||
546 | - | ||
547 | - # 插入到所有抽稀图层中 | ||
548 | - for level in range(self.max_level): | ||
549 | - | ||
550 | - center: Geometry = g.Centroid() | ||
551 | - | ||
552 | - extent = g.GetEnvelope() | ||
553 | - long_extent= extent[1]-extent[0] | ||
554 | - lat_extent = extent[3]-extent[2] | ||
555 | - | ||
556 | - this_grid_len =self.vacuate_layers_gridsize[level] | ||
557 | - #超大的直接加入 | ||
558 | - # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | ||
559 | - # vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
560 | - # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
561 | - # feat.SetGeometry(g) | ||
562 | - # vacuate_layer.CreateFeature(feat) | ||
563 | - # else: | ||
564 | - | ||
565 | - row = int((center.GetY() - self.extent[2]) / this_grid_len) | ||
566 | - col = int((center.GetX() - self.extent[0]) / this_grid_len) | ||
567 | - key = "{}.{}.{}".format(level, row, col) | ||
568 | - | ||
569 | - if not self.fill_dict.get(key): | ||
570 | - self.fill_dict[key] = 0 | ||
571 | - if self.fill_dict[key] == 0: | ||
572 | - | ||
573 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
574 | - feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
575 | - # 如果图形比网格小,直接存储其中心点 | ||
576 | - if this_grid_len>long_extent and this_grid_len>lat_extent: | ||
577 | - feat.SetGeometry(center) | ||
578 | - else: | ||
579 | - feat.SetGeometry(g) | ||
580 | - | ||
581 | - # 复制旧feature属性 | ||
582 | - field_dict = feature.items() | ||
583 | - for field_name in field_dict: | ||
584 | - feat.SetField(field_name, field_dict[field_name]) | ||
585 | - feat.SetField("_dcigrid_name_",".".join(key.split(".")[1:])) | ||
586 | - | ||
587 | - vacuate_layer.CreateFeature(feat) | ||
588 | - self.fill_dict[key] += 1 | ||
589 | - #超大的还有机会 | ||
590 | - elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | ||
591 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
592 | - feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
593 | - feat.SetGeometry(g) | ||
594 | - | ||
595 | - # 复制旧feature属性 | ||
596 | - field_dict = feature.items() | ||
597 | - for field_name in field_dict: | ||
598 | - feat.SetField(field_name, field_dict[field_name]) | ||
599 | - feat.SetField("_dcigrid_name_",".".join(key.split(".")[1:])) | ||
600 | - | ||
601 | - vacuate_layer.CreateFeature(feat) | ||
602 | - self.fill_dict[key] += 1 | ||
603 | - else: | ||
604 | - self.fill_dict[key] += 1 | ||
605 | - | ||
606 | - def set_vacuate_count(self): | ||
607 | - if self.is_spatial: | ||
608 | - # 插入到所有抽稀图层中 | ||
609 | - for level in range(self.max_level): | ||
610 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
611 | - for feat in vacuate_layer: | ||
612 | - key = "{}.{}".format(level,feat.GetField("_dcigrid_name_")) | ||
613 | - feat.SetField("_dcigrid_count_",self.fill_dict.get(key)) | ||
614 | - vacuate_layer.SetFeature(feat) | ||
615 | - | ||
616 | - def end(self): | ||
617 | - for pg in self.pg_ds_dict.values(): | ||
618 | - pg.Destroy() | ||
619 | - | ||
620 | - def rollback(self): | ||
621 | - for pg in self.pg_ds_dict.values(): | ||
622 | - pg.RollbackTransaction() |
@@ -6,8 +6,9 @@ | @@ -6,8 +6,9 @@ | ||
6 | from flask import request | 6 | from flask import request |
7 | import uuid | 7 | import uuid |
8 | import os | 8 | import os |
9 | - | ||
10 | - | 9 | +import stat |
10 | +import platform | ||
11 | +from app.util.component.StructuredPrint import StructurePrint | ||
11 | class FileProcess: | 12 | class FileProcess: |
12 | @classmethod | 13 | @classmethod |
13 | def save(cls,parent): | 14 | def save(cls,parent): |
@@ -21,9 +22,15 @@ class FileProcess: | @@ -21,9 +22,15 @@ class FileProcess: | ||
21 | os.makedirs(dir_path) | 22 | os.makedirs(dir_path) |
22 | #有时候文件最后会多一个" | 23 | #有时候文件最后会多一个" |
23 | filename = file.filename.split('"')[0] | 24 | filename = file.filename.split('"')[0] |
25 | + | ||
24 | store_file = os.path.join(dir_path, filename) | 26 | store_file = os.path.join(dir_path, filename) |
27 | + StructurePrint().print(store_file) | ||
28 | + | ||
25 | file.save(store_file) | 29 | file.save(store_file) |
26 | 30 | ||
31 | + if platform.system().lower() == 'linux': | ||
32 | + #设置为444权限 | ||
33 | + os.chmod(store_file, stat.S_IRUSR + stat.S_IRGRP + stat.S_IROTH) | ||
27 | return dir_path, store_file | 34 | return dir_path, store_file |
28 | 35 | ||
29 | @classmethod | 36 | @classmethod |
@@ -36,6 +36,7 @@ class ZipUtil: | @@ -36,6 +36,7 @@ class ZipUtil: | ||
36 | name_t = names.encode('cp437').decode('gbk') | 36 | name_t = names.encode('cp437').decode('gbk') |
37 | except: | 37 | except: |
38 | name_t = names.encode('utf-8').decode('utf-8') | 38 | name_t = names.encode('utf-8').decode('utf-8') |
39 | + # name_t = names | ||
39 | 40 | ||
40 | zip_file.extract(names,store_path) | 41 | zip_file.extract(names,store_path) |
41 | 42 |
envvars
0 → 100644
1 | +# envvars - default environment variables for apache2ctl | ||
2 | + | ||
3 | +# this won't be correct after changing uid | ||
4 | +unset HOME | ||
5 | + | ||
6 | +# for supporting multiple apache2 instances | ||
7 | +if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then | ||
8 | + SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}" | ||
9 | +else | ||
10 | + SUFFIX= | ||
11 | +fi | ||
12 | + | ||
13 | +# Since there is no sane way to get the parsed apache2 config in scripts, some | ||
14 | +# settings are defined via environment variables and then used in apache2ctl, | ||
15 | +# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc. | ||
16 | +export APACHE_RUN_USER=www-data | ||
17 | +export APACHE_RUN_GROUP=www-data | ||
18 | +# temporary state file location. This might be changed to /run in Wheezy+1 | ||
19 | +export APACHE_PID_FILE=/var/run/apache2$SUFFIX/apache2.pid | ||
20 | +export APACHE_RUN_DIR=/var/run/apache2$SUFFIX | ||
21 | +export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX | ||
22 | +# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. | ||
23 | +export APACHE_LOG_DIR=/var/log/apache2$SUFFIX | ||
24 | + | ||
25 | +## The locale used by some modules like mod_dav | ||
26 | +export LANG="C.UTF-8" | ||
27 | +## Uncomment the following line to use the system default locale instead: | ||
28 | +export LANG | ||
29 | + | ||
30 | +export TZ="Asia/Shanghai" | ||
31 | + | ||
32 | +## The command to get the status for 'apache2ctl status'. | ||
33 | +## Some packages providing 'www-browser' need '--dump' instead of '-dump'. | ||
34 | +#export APACHE_LYNX='www-browser -dump' | ||
35 | + | ||
36 | +## If you need a higher file descriptor limit, uncomment and adjust the | ||
37 | +## following line (default is 8192): | ||
38 | +#APACHE_ULIMIT_MAX_FILES='ulimit -n 65536' | ||
39 | + | ||
40 | +## If you would like to pass arguments to the web server, add them below | ||
41 | +## to the APACHE_ARGUMENTS environment. | ||
42 | +#export APACHE_ARGUMENTS='' | ||
43 | + | ||
44 | +## Enable the debug mode for maintainer scripts. | ||
45 | +## This will produce a verbose output on package installations of web server modules and web application | ||
46 | +## installations which interact with Apache | ||
47 | +#export APACHE2_MAINTSCRIPT_DEBUG=1 |
1 | -flask==1.1.2 | ||
2 | -SQLAlchemy==1.3.17 | ||
3 | -Flask-SQLAlchemy==2.4.3 | ||
4 | -gevent==20.9.0 | ||
5 | -gunicorn==20.0.4 | ||
6 | -flask_cors==3.0.8 | ||
7 | -flasgger==0.9.5 | ||
8 | -#GDAL==3.2.1 | ||
9 | -psycopg2-binary==2.8.5 | ||
10 | -pyDes==2.0.1 | ||
11 | -gevent-websocket==0.10.1 | ||
12 | -opencv-python==4.5.1.48 | ||
13 | -mod_wsgi==4.8.0 | ||
14 | -thrift==0.13.0 | ||
15 | -Authlib==0.13 | ||
16 | -kazoo==2.8.0 | ||
17 | -paramiko==2.8.0 | ||
18 | -requests==2.26.0 | ||
19 | -schedule==1.1.0 | 1 | +flask==1.1.2 |
2 | +SQLAlchemy==1.3.17 | ||
3 | +Flask-SQLAlchemy==2.4.3 | ||
4 | +gevent==20.9.0 | ||
5 | +gunicorn==20.0.4 | ||
6 | +flask_cors==3.0.8 | ||
7 | +flasgger==0.9.5 | ||
8 | +#GDAL==3.2.1 | ||
9 | +psycopg2-binary==2.8.5 | ||
10 | +pyDes==2.0.1 | ||
11 | +gevent-websocket==0.10.1 | ||
12 | +opencv-python==4.5.1.48 | ||
13 | +mod_wsgi==4.8.0 | ||
14 | +thrift==0.13.0 | ||
15 | +Authlib==0.13 | ||
16 | +kazoo==2.8.0 | ||
17 | +paramiko==2.8.0 | ||
18 | +requests==2.26.0 | ||
19 | +schedule==1.1.0 |
@@ -16,7 +16,7 @@ else | @@ -16,7 +16,7 @@ else | ||
16 | echo "端口设置为$1 ..." | 16 | echo "端口设置为$1 ..." |
17 | fi | 17 | fi |
18 | 18 | ||
19 | -docker run -d --name $dn -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -p $port:8840 -v $curPath:/usr/src/app -w /usr/src/app dci/dmapdms:3.0 python3 ./run.py | 19 | +docker run -d --name $dn -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -p $port:8840 -v $curPath:/usr/src/app -w /usr/src/app dci/dmapserver:build python3 ./run.py |
20 | # 清除未完成的任务任务 | 20 | # 清除未完成的任务任务 |
21 | sleep 5 | 21 | sleep 5 |
22 | curl localhost:$port/release | 22 | curl localhost:$port/release |
@@ -33,8 +33,8 @@ fi | @@ -33,8 +33,8 @@ fi | ||
33 | 33 | ||
34 | #启动容器和apache | 34 | #启动容器和apache |
35 | echo "正在启动容器..." | 35 | echo "正在启动容器..." |
36 | -set="--privileged=true -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0" | ||
37 | -docker run -it -d --name $container_name $set -p ${port}:80 -p ${port2}:81 -v ${curPath}:/usr/src/app -v ${curPath}/apache2.conf:/etc/apache2/apache2.conf -v ${curPath}/dmapmanager.conf:/etc/apache2/sites-enabled/dmapmanager.conf dci/dmapserver:base | 36 | +set="--privileged=true -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -e LAND=C.UTF-8" |
37 | +docker run -it -d --name ${container_name} ${set} -p ${port}:80 -p ${port2}:81 -v ${curPath}:/usr/src/app -v ${curPath}/apache2.conf:/etc/apache2/apache2.conf -v ${curPath}/dmapmanager.conf:/etc/apache2/sites-enabled/dmapmanager.conf -v ${curPath}/envvars:/etc/apache2/envvars dci/dmapserver:build | ||
38 | docker exec -d $container_name service apache2 start | 38 | docker exec -d $container_name service apache2 start |
39 | sleep 5 | 39 | sleep 5 |
40 | curl localhost:$port/release | 40 | curl localhost:$port/release |
请
注册
或
登录
后发表评论