正在显示
24 个修改的文件
包含
198 行增加
和
689 行删除
| @@ -7,23 +7,15 @@ ENV LANG=en_US.UTF-8 | @@ -7,23 +7,15 @@ ENV LANG=en_US.UTF-8 | ||
| 7 | #设置时区 | 7 | #设置时区 |
| 8 | ENV TZ=Asia/Shanghai | 8 | ENV TZ=Asia/Shanghai |
| 9 | 9 | ||
| 10 | +COPY . . | ||
| 10 | 11 | ||
| 11 | RUN apt-get update | 12 | RUN apt-get update |
| 12 | -#安装pip | 13 | +#安装pip3 |
| 13 | RUN apt-get install python3-pip -y | 14 | RUN apt-get install python3-pip -y |
| 14 | - | ||
| 15 | -#安装opencv所需依赖库 | ||
| 16 | -RUN apt-get install libgl1-mesa-glx -y | ||
| 17 | -RUN apt-get install libglib2.0-dev -y | ||
| 18 | - | ||
| 19 | #安装apache2 | 15 | #安装apache2 |
| 20 | -RUN apt-get install apache2 -y | ||
| 21 | -RUN apt-get install apache2-dev -y | ||
| 22 | - | ||
| 23 | - | 16 | +RUN apt-get install apache2 -y && apt-get install apache2-dev -y |
| 24 | #安装依赖 | 17 | #安装依赖 |
| 25 | -COPY . . | ||
| 26 | RUN pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ --trusted-host pypi.tuna.tsinghua.edu.cn | 18 | RUN pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple/ --trusted-host pypi.tuna.tsinghua.edu.cn |
| 27 | 19 | ||
| 20 | +RUN mod_wsgi-express install-module | ||
| 28 | 21 | ||
| 29 | -RUN mod_wsgi-express install-module |
BuildImage/requirements.txt
0 → 100644
| 1 | +flask==1.1.2 | ||
| 2 | +SQLAlchemy==1.3.17 | ||
| 3 | +Flask-SQLAlchemy==2.4.3 | ||
| 4 | +flask_cors==3.0.8 | ||
| 5 | +flasgger==0.9.5 | ||
| 6 | +psycopg2-binary==2.8.5 | ||
| 7 | +pyDes==2.0.1 | ||
| 8 | +mod_wsgi==4.8.0 | ||
| 9 | +opencv-python-headless==4.5.1.48 | ||
| 10 | +thrift==0.13.0 | ||
| 11 | +Authlib==0.13 | ||
| 12 | +kazoo==2.8.0 | ||
| 13 | +paramiko==2.8.0 | ||
| 14 | +requests==2.26.0 | ||
| 15 | +schedule==1.1.0 |
BuildImage/second-build/Dockerfile
0 → 100644
| 1 | +From osgeo/gdal:ubuntu-small-3.4.1 | ||
| 2 | +WORKDIR /root | ||
| 3 | +#设置编码 | ||
| 4 | +ENV LANG=en_US.UTF-8 | ||
| 5 | +#设置时区 | ||
| 6 | +ENV TZ=Asia/Shanghai | ||
| 7 | + | ||
| 8 | +#安装apache2 | ||
| 9 | +RUN apt-get update | ||
| 10 | +RUN apt-get install apache2 -y | ||
| 11 | + | ||
| 12 | +COPY --from=dci/dmapserver:build-dev /usr/local /usr/local | ||
| 13 | +COPY --from=dci/dmapserver:build-dev /usr/lib/apache2/modules /usr/lib/apache2/modules | ||
| 14 | + | ||
| 15 | + |
| @@ -16,6 +16,10 @@ from app.util.component.StructuredPrint import StructurePrint | @@ -16,6 +16,10 @@ from app.util.component.StructuredPrint import StructurePrint | ||
| 16 | from app.util.component.PGUtil import PGUtil | 16 | from app.util.component.PGUtil import PGUtil |
| 17 | import os | 17 | import os |
| 18 | from app.modules.monitor.schedule import start_schedule | 18 | from app.modules.monitor.schedule import start_schedule |
| 19 | +import datetime | ||
| 20 | + | ||
| 21 | + | ||
| 22 | + | ||
| 19 | 23 | ||
| 20 | 24 | ||
| 21 | class JSONEncoder(_JSONEncoder): | 25 | class JSONEncoder(_JSONEncoder): |
| @@ -76,11 +80,13 @@ def create_app(): | @@ -76,11 +80,13 @@ def create_app(): | ||
| 76 | db.create_all(app=app) | 80 | db.create_all(app=app) |
| 77 | 81 | ||
| 78 | # 日志 | 82 | # 日志 |
| 83 | + | ||
| 79 | logging.basicConfig(level=configure.log_level) | 84 | logging.basicConfig(level=configure.log_level) |
| 80 | log_file = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "logs", "log.txt") | 85 | log_file = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "logs", "log.txt") |
| 81 | handler = logging.FileHandler(log_file, encoding='UTF-8') # 设置日志字符集和存储路径名字 | 86 | handler = logging.FileHandler(log_file, encoding='UTF-8') # 设置日志字符集和存储路径名字 |
| 82 | logging_format = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s') | 87 | logging_format = logging.Formatter('[%(levelname)s] %(asctime)s %(message)s') |
| 83 | handler.setFormatter(logging_format) | 88 | handler.setFormatter(logging_format) |
| 89 | + | ||
| 84 | app.logger.addHandler(handler) | 90 | app.logger.addHandler(handler) |
| 85 | 91 | ||
| 86 | # 配置使用鉴权组件,不写无法认证授权 | 92 | # 配置使用鉴权组件,不写无法认证授权 |
| @@ -98,7 +98,7 @@ class DataManager(BlueprintApi): | @@ -98,7 +98,7 @@ class DataManager(BlueprintApi): | ||
| 98 | return database_info.Api().result | 98 | return database_info.Api().result |
| 99 | 99 | ||
| 100 | @staticmethod | 100 | @staticmethod |
| 101 | - @bp.route('/Detail', methods=["POST"]) | 101 | + @bp.route('/Detail', methods=["GET","POST"]) |
| 102 | @swag_from(database_detail.Api.api_doc) | 102 | @swag_from(database_detail.Api.api_doc) |
| 103 | def api_database_detail(): | 103 | def api_database_detail(): |
| 104 | """ | 104 | """ |
| @@ -13,14 +13,20 @@ from app.util.component.PGUtil import PGUtil | @@ -13,14 +13,20 @@ from app.util.component.PGUtil import PGUtil | ||
| 13 | import configure | 13 | import configure |
| 14 | from osgeo.ogr import DataSource | 14 | from osgeo.ogr import DataSource |
| 15 | from flask import current_app | 15 | from flask import current_app |
| 16 | +from authlib.integrations.flask_oauth2 import current_token | ||
| 17 | + | ||
| 18 | + | ||
| 16 | class Api(ApiTemplate): | 19 | class Api(ApiTemplate): |
| 17 | api_name = "删除数据库" | 20 | api_name = "删除数据库" |
| 18 | def process(self): | 21 | def process(self): |
| 19 | re ={} | 22 | re ={} |
| 20 | va_ds = None | 23 | va_ds = None |
| 21 | try: | 24 | try: |
| 22 | - | 25 | + |
| 23 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() | 26 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() |
| 27 | + if database.creator != current_token.user.username: | ||
| 28 | + raise Exception("缺乏权限!") | ||
| 29 | + | ||
| 24 | if configure.VACUATE_DB_URI: | 30 | if configure.VACUATE_DB_URI: |
| 25 | va_ds: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) | 31 | va_ds: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) |
| 26 | else: | 32 | else: |
| @@ -62,7 +68,7 @@ class Api(ApiTemplate): | @@ -62,7 +68,7 @@ class Api(ApiTemplate): | ||
| 62 | "parameters":[ | 68 | "parameters":[ |
| 63 | {"name": "guid", | 69 | {"name": "guid", |
| 64 | "in": "formData", | 70 | "in": "formData", |
| 65 | - "type": "string","description":"数据库guid","required": "true"}, | 71 | + "type": "string","description":"数据库guid","required": "true"} |
| 66 | 72 | ||
| 67 | ], | 73 | ], |
| 68 | "responses":{ | 74 | "responses":{ |
| @@ -28,6 +28,9 @@ class Api(ApiTemplate): | @@ -28,6 +28,9 @@ class Api(ApiTemplate): | ||
| 28 | database = Database.query.filter_by(guid=guid) | 28 | database = Database.query.filter_by(guid=guid) |
| 29 | dbase:Database = database.one_or_none() | 29 | dbase:Database = database.one_or_none() |
| 30 | 30 | ||
| 31 | + if dbase.creator != "": | ||
| 32 | + raise Exception("缺乏权限!") | ||
| 33 | + | ||
| 31 | update_dict={} | 34 | update_dict={} |
| 32 | 35 | ||
| 33 | if not dbase: | 36 | if not dbase: |
| @@ -18,7 +18,7 @@ from sqlalchemy.orm import Session | @@ -18,7 +18,7 @@ from sqlalchemy.orm import Session | ||
| 18 | import configure | 18 | import configure |
| 19 | import datetime | 19 | import datetime |
| 20 | import multiprocessing | 20 | import multiprocessing |
| 21 | -from ..util.EntryDataVacuate import EntryDataVacuate | 21 | +from .util.EntryDataVacuate import EntryDataVacuate |
| 22 | from app.util.component.TaskController import TaskController | 22 | from app.util.component.TaskController import TaskController |
| 23 | from app.util.component.TaskWriter import TaskWriter | 23 | from app.util.component.TaskWriter import TaskWriter |
| 24 | class Api(ApiTemplate): | 24 | class Api(ApiTemplate): |
| @@ -212,6 +212,13 @@ class Api(ApiTemplate): | @@ -212,6 +212,13 @@ class Api(ApiTemplate): | ||
| 212 | "in": "formData", | 212 | "in": "formData", |
| 213 | "type": "string", | 213 | "type": "string", |
| 214 | "description": "目录guid"}, | 214 | "description": "目录guid"}, |
| 215 | + | ||
| 216 | + {"name": "vacuate", | ||
| 217 | + "in": "formData", | ||
| 218 | + "type": "string", | ||
| 219 | + "description": "是否抽稀", | ||
| 220 | + "enum":[1,0]}, | ||
| 221 | + | ||
| 215 | {"name": "check_meta_only", | 222 | {"name": "check_meta_only", |
| 216 | "in": "formData", | 223 | "in": "formData", |
| 217 | "type": "int", | 224 | "type": "int", |
| @@ -15,7 +15,7 @@ import json | @@ -15,7 +15,7 @@ import json | ||
| 15 | from app.util.component.ApiTemplate import ApiTemplate | 15 | from app.util.component.ApiTemplate import ApiTemplate |
| 16 | from app.util.component.ZipUtil import ZipUtil | 16 | from app.util.component.ZipUtil import ZipUtil |
| 17 | from app.util.component.FileProcess import FileProcess | 17 | from app.util.component.FileProcess import FileProcess |
| 18 | - | 18 | +import platform |
| 19 | 19 | ||
| 20 | class Api(ApiTemplate): | 20 | class Api(ApiTemplate): |
| 21 | api_name = "获取meta" | 21 | api_name = "获取meta" |
| @@ -53,7 +53,12 @@ class Api(ApiTemplate): | @@ -53,7 +53,12 @@ class Api(ApiTemplate): | ||
| 53 | 53 | ||
| 54 | else: | 54 | else: |
| 55 | # 保存文件 | 55 | # 保存文件 |
| 56 | - parent = os.path.dirname(os.path.realpath(__file__)) | 56 | + # 保存在项目外 |
| 57 | + if platform.system().lower() == 'windows': | ||
| 58 | + parent = "C:/" | ||
| 59 | + else: | ||
| 60 | + parent = "/tmp" | ||
| 61 | + | ||
| 57 | dir_path, store_file = FileProcess.save(parent) | 62 | dir_path, store_file = FileProcess.save(parent) |
| 58 | store_path = ZipUtil.unzip(store_file) | 63 | store_path = ZipUtil.unzip(store_file) |
| 59 | 64 |
| @@ -18,7 +18,6 @@ import datetime | @@ -18,7 +18,6 @@ import datetime | ||
| 18 | class EntryDataVacuate: | 18 | class EntryDataVacuate: |
| 19 | 19 | ||
| 20 | def entry(self,parameter): | 20 | def entry(self,parameter): |
| 21 | - # meta:dict = parameter.get("meta") | ||
| 22 | 21 | ||
| 23 | # 初始化任务 | 22 | # 初始化任务 |
| 24 | this_task = ThisTask(parameter) | 23 | this_task = ThisTask(parameter) |
| @@ -28,6 +27,9 @@ class EntryDataVacuate: | @@ -28,6 +27,9 @@ class EntryDataVacuate: | ||
| 28 | _data_path=None | 27 | _data_path=None |
| 29 | try: | 28 | try: |
| 30 | metas: list = parameter.get("meta") | 29 | metas: list = parameter.get("meta") |
| 30 | + | ||
| 31 | + vacuate = int(parameter.get("vacuate",1)) | ||
| 32 | + | ||
| 31 | # 总的入库是否成功 | 33 | # 总的入库是否成功 |
| 32 | is_success=True | 34 | is_success=True |
| 33 | 35 | ||
| @@ -75,7 +77,10 @@ class EntryDataVacuate: | @@ -75,7 +77,10 @@ class EntryDataVacuate: | ||
| 75 | if is_success: | 77 | if is_success: |
| 76 | # 更新任务为成功任务 | 78 | # 更新任务为成功任务 |
| 77 | this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) | 79 | this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) |
| 78 | - this_task.commit() | 80 | + try: |
| 81 | + this_task.commit() | ||
| 82 | + except: | ||
| 83 | + raise Exception("可能编码出错!") | ||
| 79 | else: | 84 | else: |
| 80 | # 更新任务为失败任务 | 85 | # 更新任务为失败任务 |
| 81 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | 86 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) |
| @@ -83,6 +88,7 @@ class EntryDataVacuate: | @@ -83,6 +88,7 @@ class EntryDataVacuate: | ||
| 83 | this_task.rollback() | 88 | this_task.rollback() |
| 84 | 89 | ||
| 85 | except Exception as e: | 90 | except Exception as e: |
| 91 | + StructurePrint().print("herehere") | ||
| 86 | this_task.write_process("{} 任务结束!".format(e.__str__())) | 92 | this_task.write_process("{} 任务结束!".format(e.__str__())) |
| 87 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | 93 | this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) |
| 88 | StructurePrint().print(e.__str__(),"ERROR") | 94 | StructurePrint().print(e.__str__(),"ERROR") |
| @@ -98,7 +104,7 @@ class EntryDataVacuate: | @@ -98,7 +104,7 @@ class EntryDataVacuate: | ||
| 98 | dir_path = os.path.dirname(dir_path) | 104 | dir_path = os.path.dirname(dir_path) |
| 99 | i+=1 | 105 | i+=1 |
| 100 | if i<30: | 106 | if i<30: |
| 101 | - shutil.rmtree(dir_path,True) | 107 | + # shutil.rmtree(dir_path,True) |
| 102 | StructurePrint().print("删除文件成功!") | 108 | StructurePrint().print("删除文件成功!") |
| 103 | else: | 109 | else: |
| 104 | raise Exception("找不到文件!") | 110 | raise Exception("找不到文件!") |
| @@ -156,15 +162,18 @@ class EntryDataVacuate: | @@ -156,15 +162,18 @@ class EntryDataVacuate: | ||
| 156 | 162 | ||
| 157 | def entry_one_layer(self,layer: Layer,this_task,meta): | 163 | def entry_one_layer(self,layer: Layer,this_task,meta): |
| 158 | 164 | ||
| 159 | - # this_task.pg_ds.StartTransaction() | 165 | + |
| 160 | new_layer_name = None | 166 | new_layer_name = None |
| 161 | - # vacuate_process= None | 167 | + vacuate_process= None |
| 168 | + vacuate = int(this_task.parameter.get("vacuate", 1)) | ||
| 162 | success = True | 169 | success = True |
| 163 | table_guid = uuid.uuid1().__str__() | 170 | table_guid = uuid.uuid1().__str__() |
| 164 | try: | 171 | try: |
| 165 | # 图层设置 | 172 | # 图层设置 |
| 166 | parameter = this_task.parameter | 173 | parameter = this_task.parameter |
| 167 | 174 | ||
| 175 | + | ||
| 176 | + | ||
| 168 | overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" | 177 | overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" |
| 169 | geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" | 178 | geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" |
| 170 | fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" | 179 | fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" |
| @@ -201,9 +210,8 @@ class EntryDataVacuate: | @@ -201,9 +210,8 @@ class EntryDataVacuate: | ||
| 201 | pg_layer.CreateFields(schema) | 210 | pg_layer.CreateFields(schema) |
| 202 | 211 | ||
| 203 | #创建抽稀过程 | 212 | #创建抽稀过程 |
| 204 | - # vacuate_process = VacuateProcess(layer,table_guid,options) | ||
| 205 | - | ||
| 206 | - | 213 | + if vacuate: |
| 214 | + vacuate_process = VacuateProcess(layer,table_guid,options) | ||
| 207 | 215 | ||
| 208 | count =0 | 216 | count =0 |
| 209 | 217 | ||
| @@ -231,18 +239,18 @@ class EntryDataVacuate: | @@ -231,18 +239,18 @@ class EntryDataVacuate: | ||
| 231 | pg_layer.CreateFeature(out_feature) | 239 | pg_layer.CreateFeature(out_feature) |
| 232 | 240 | ||
| 233 | #插入抽稀图层 | 241 | #插入抽稀图层 |
| 234 | - # if out_geom is not None: | ||
| 235 | - # vacuate_process.vacuate(out_geom) | 242 | + if out_geom is not None and vacuate: |
| 243 | + vacuate_process.vacuate(out_geom) | ||
| 236 | 244 | ||
| 237 | # 注册图层信息 | 245 | # 注册图层信息 |
| 238 | # 是否抽吸过 | 246 | # 是否抽吸过 |
| 239 | - # is_vacuate = 1 if vacuate_process.max_level>0 else 0 | ||
| 240 | - is_vacuate = 0 | 247 | + is_vacuate = 1 if vacuate_process.max_level>0 else 0 |
| 241 | 248 | ||
| 242 | this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | 249 | this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) |
| 243 | 250 | ||
| 244 | # 注册抽稀表 | 251 | # 注册抽稀表 |
| 245 | - # this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | 252 | + if vacuate: |
| 253 | + this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | ||
| 246 | 254 | ||
| 247 | this_task.write_process("{}图层入库成功。".format(new_layer_name)) | 255 | this_task.write_process("{}图层入库成功。".format(new_layer_name)) |
| 248 | 256 | ||
| @@ -252,11 +260,13 @@ class EntryDataVacuate: | @@ -252,11 +260,13 @@ class EntryDataVacuate: | ||
| 252 | StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") | 260 | StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") |
| 253 | print(traceback.format_exc()) | 261 | print(traceback.format_exc()) |
| 254 | # 抽稀回滚 | 262 | # 抽稀回滚 |
| 255 | - # vacuate_process.rollback() | 263 | + if vacuate: |
| 264 | + vacuate_process.rollback() | ||
| 256 | success =False | 265 | success =False |
| 257 | 266 | ||
| 258 | finally: | 267 | finally: |
| 259 | - # vacuate_process.end() | 268 | + if vacuate: |
| 269 | + vacuate_process.end() | ||
| 260 | pass | 270 | pass |
| 261 | return success,new_layer_name | 271 | return success,new_layer_name |
| 262 | 272 | ||
| @@ -459,12 +469,6 @@ class VacuateProcess: | @@ -459,12 +469,6 @@ class VacuateProcess: | ||
| 459 | # 固有疏密程度 | 469 | # 固有疏密程度 |
| 460 | original_density=8 | 470 | original_density=8 |
| 461 | 471 | ||
| 462 | - | ||
| 463 | - # 额外一层 | ||
| 464 | - # self.this_gridsize.append(0.000075) | ||
| 465 | - # self.max_level += 1 | ||
| 466 | - ###### | ||
| 467 | - | ||
| 468 | if extent[0]>180: | 472 | if extent[0]>180: |
| 469 | self.t_grid_size=self.project_gridsize | 473 | self.t_grid_size=self.project_gridsize |
| 470 | else: | 474 | else: |
| @@ -19,6 +19,14 @@ class Api(ApiTemplate): | @@ -19,6 +19,14 @@ class Api(ApiTemplate): | ||
| 19 | 19 | ||
| 20 | column_guid = self.para.get("column_guid") | 20 | column_guid = self.para.get("column_guid") |
| 21 | update_dict = {} | 21 | update_dict = {} |
| 22 | + | ||
| 23 | + column:Columns = Columns.query.filter_by(guid = column_guid).one_or_none() | ||
| 24 | + if not column: | ||
| 25 | + raise Exception("数据不存在!") | ||
| 26 | + | ||
| 27 | + if column.relate_table.relate_database.creator != "": | ||
| 28 | + raise Exception("缺乏权限!") | ||
| 29 | + | ||
| 22 | try: | 30 | try: |
| 23 | if self.para.get("column_alias"): | 31 | if self.para.get("column_alias"): |
| 24 | update_dict["alias"] = self.para.get("column_alias") | 32 | update_dict["alias"] = self.para.get("column_alias") |
| @@ -47,7 +47,10 @@ class Api(ApiTemplate): | @@ -47,7 +47,10 @@ class Api(ApiTemplate): | ||
| 47 | res["msg"]= "数据库不存在!" | 47 | res["msg"]= "数据库不存在!" |
| 48 | return res | 48 | return res |
| 49 | 49 | ||
| 50 | - | 50 | + if database.creator != "": |
| 51 | + raise Exception("缺乏权限!") | ||
| 52 | + | ||
| 53 | + | ||
| 51 | pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri)) | 54 | pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri)) |
| 52 | 55 | ||
| 53 | if configure.VACUATE_DB_URI: | 56 | if configure.VACUATE_DB_URI: |
| @@ -34,6 +34,8 @@ class Api(ApiTemplate): | @@ -34,6 +34,8 @@ class Api(ApiTemplate): | ||
| 34 | res["msg"]= "数据不存在!" | 34 | res["msg"]= "数据不存在!" |
| 35 | return res | 35 | return res |
| 36 | 36 | ||
| 37 | + if table.relate_database.creator != "": | ||
| 38 | + raise Exception("缺乏权限!") | ||
| 37 | 39 | ||
| 38 | if self.para.__contains__("catalog_guid"): | 40 | if self.para.__contains__("catalog_guid"): |
| 39 | if catalog_guid is None: | 41 | if catalog_guid is None: |
| @@ -21,9 +21,10 @@ class Api(ApiTemplate): | @@ -21,9 +21,10 @@ class Api(ApiTemplate): | ||
| 21 | raise Exception("数据不存在!") | 21 | raise Exception("数据不存在!") |
| 22 | pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 22 | pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
| 23 | layer:Layer = pg_ds.GetLayerByName(table.name) | 23 | layer:Layer = pg_ds.GetLayerByName(table.name) |
| 24 | - append_dict ={"srid":None,"sr_wkt":None,"sr_proj4":None,"exist":1} | 24 | + append_dict ={"srid":None,"sr_wkt":None,"sr_proj4":None,"exist":1,"schema":"public","geomfield":layer.GetGeometryColumn()} |
| 25 | if layer: | 25 | if layer: |
| 26 | sr:SpatialReference = layer.GetSpatialRef() | 26 | sr:SpatialReference = layer.GetSpatialRef() |
| 27 | + | ||
| 27 | if sr: | 28 | if sr: |
| 28 | append_dict["srid"] = sr.GetAuthorityCode(None) | 29 | append_dict["srid"] = sr.GetAuthorityCode(None) |
| 29 | append_dict["sr_wkt"] = sr.ExportToWkt() | 30 | append_dict["sr_wkt"] = sr.ExportToWkt() |
| @@ -32,6 +32,9 @@ class Api(ApiTemplate): | @@ -32,6 +32,9 @@ class Api(ApiTemplate): | ||
| 32 | database_guid = self.para.get("database_guid") | 32 | database_guid = self.para.get("database_guid") |
| 33 | database = Database.query.filter_by(guid=database_guid).one_or_none() | 33 | database = Database.query.filter_by(guid=database_guid).one_or_none() |
| 34 | 34 | ||
| 35 | + if database.creator != "": | ||
| 36 | + raise Exception("缺乏权限!") | ||
| 37 | + | ||
| 35 | if not database: | 38 | if not database: |
| 36 | raise Exception("数据库不存在!") | 39 | raise Exception("数据库不存在!") |
| 37 | # 初始化task | 40 | # 初始化task |
| @@ -42,6 +42,9 @@ class Api(ApiTemplate): | @@ -42,6 +42,9 @@ class Api(ApiTemplate): | ||
| 42 | if not table: | 42 | if not table: |
| 43 | raise Exception("数据不存在!") | 43 | raise Exception("数据不存在!") |
| 44 | 44 | ||
| 45 | + if table.relate_database.creator != "": | ||
| 46 | + raise Exception("缺乏权限!") | ||
| 47 | + | ||
| 45 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 48 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
| 46 | layer = pg_ds.GetLayerByName(table.name) | 49 | layer = pg_ds.GetLayerByName(table.name) |
| 47 | # 判断图层是否存在 | 50 | # 判断图层是否存在 |
| @@ -73,7 +76,7 @@ class Api(ApiTemplate): | @@ -73,7 +76,7 @@ class Api(ApiTemplate): | ||
| 73 | 76 | ||
| 74 | 77 | ||
| 75 | task = Task(guid=task_guid, | 78 | task = Task(guid=task_guid, |
| 76 | - name="{}矢量金字塔构建".format(table.name), | 79 | + name="{}构建矢量金字塔".format(table.name), |
| 77 | table_guid=table_guid, | 80 | table_guid=table_guid, |
| 78 | create_time=datetime.datetime.now(), | 81 | create_time=datetime.datetime.now(), |
| 79 | state=0, | 82 | state=0, |
| @@ -40,6 +40,9 @@ class Api(ApiTemplate): | @@ -40,6 +40,9 @@ class Api(ApiTemplate): | ||
| 40 | if not table: | 40 | if not table: |
| 41 | raise Exception("数据不存在!") | 41 | raise Exception("数据不存在!") |
| 42 | 42 | ||
| 43 | + if table.relate_database.creator != "": | ||
| 44 | + raise Exception("缺乏权限!") | ||
| 45 | + | ||
| 43 | # 判断图层是否存在 | 46 | # 判断图层是否存在 |
| 44 | 47 | ||
| 45 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) | 48 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
| @@ -73,7 +76,7 @@ class Api(ApiTemplate): | @@ -73,7 +76,7 @@ class Api(ApiTemplate): | ||
| 73 | 76 | ||
| 74 | 77 | ||
| 75 | task = Task(guid=task_guid, | 78 | task = Task(guid=task_guid, |
| 76 | - name="{}矢量金字塔构建,网格大小:{}".format(table.name,self.para.get("grids")), | 79 | + name="{}构建矢量金字塔,网格大小:{}".format(table.name,self.para.get("grids")), |
| 77 | table_guid=table_guid, | 80 | table_guid=table_guid, |
| 78 | create_time=datetime.datetime.now(), | 81 | create_time=datetime.datetime.now(), |
| 79 | state=0, | 82 | state=0, |
| 1 | - | ||
| 2 | -from osgeo.ogr import * | ||
| 3 | -from osgeo import ogr | ||
| 4 | -from osgeo import gdal | ||
| 5 | -import os | ||
| 6 | -import uuid | ||
| 7 | -import shutil | ||
| 8 | -import time | ||
| 9 | -from app.modules.data.models import * | ||
| 10 | -from app.util.component.PGUtil import PGUtil | ||
| 11 | -from app.util.component.StructuredPrint import StructurePrint | ||
| 12 | -from sqlalchemy.orm import Session | ||
| 13 | -import configure | ||
| 14 | -import math | ||
| 15 | -from functools import lru_cache | ||
| 16 | -import traceback | ||
| 17 | -import copy | ||
| 18 | -from app.util.component.GeometryAdapter import GeometryAdapter | ||
| 19 | -from app.util.component.VacuateConf import VacuateConf | ||
| 20 | -import datetime | ||
| 21 | -class EntryDataVacuate: | ||
| 22 | - | ||
| 23 | - def entry(self,parameter): | ||
| 24 | - # meta:dict = parameter.get("meta") | ||
| 25 | - | ||
| 26 | - # 初始化任务 | ||
| 27 | - this_task = ThisTask(parameter) | ||
| 28 | - this_task.write_process("入库任务初始化...") | ||
| 29 | - | ||
| 30 | - # 数据路径,用作删除 | ||
| 31 | - _data_path=None | ||
| 32 | - try: | ||
| 33 | - metas: list = parameter.get("meta") | ||
| 34 | - # 总的入库是否成功 | ||
| 35 | - is_success=True | ||
| 36 | - | ||
| 37 | - this_task.update({"process": "入库中"}) | ||
| 38 | - | ||
| 39 | - # 开始入库事务 | ||
| 40 | - this_task.start() | ||
| 41 | - | ||
| 42 | - # 多个文件依次入库 | ||
| 43 | - for meta in metas: | ||
| 44 | - #设置编码 | ||
| 45 | - encoding = parameter.get("encoding") | ||
| 46 | - if encoding: | ||
| 47 | - gdal.SetConfigOption("SHAPE_ENCODING",encoding) | ||
| 48 | - else: | ||
| 49 | - gdal.SetConfigOption("SHAPE_ENCODING", "GBK") | ||
| 50 | - | ||
| 51 | - #如果包含cpg文件,优先使用cpg文件中声明的编码 | ||
| 52 | - encoding_cpg = meta.get("encoding") | ||
| 53 | - if encoding_cpg: | ||
| 54 | - gdal.SetConfigOption("SHAPE_ENCODING", encoding_cpg) | ||
| 55 | - | ||
| 56 | - data_path = meta.get("data_path") | ||
| 57 | - | ||
| 58 | - #设定删除路径 | ||
| 59 | - if not _data_path: | ||
| 60 | - _data_path=data_path | ||
| 61 | - | ||
| 62 | - | ||
| 63 | - if not data_path: | ||
| 64 | - raise Exception("数据错误!") | ||
| 65 | - # 分为shp和gdb 2种录入形式 | ||
| 66 | - | ||
| 67 | - if data_path.endswith("shp"): | ||
| 68 | - is_success_one,new_layer_name = self.entry_shp(data_path,this_task,meta) | ||
| 69 | - else: | ||
| 70 | - is_success_one,new_layer_names = self.entry_gdb(data_path,this_task,meta) | ||
| 71 | - | ||
| 72 | - #如果其中一个失败,总的入库就失败 | ||
| 73 | - if not is_success_one: | ||
| 74 | - is_success=False | ||
| 75 | - | ||
| 76 | - this_task.write_process("数据入库结束。") | ||
| 77 | - | ||
| 78 | - if is_success: | ||
| 79 | - # 更新任务为成功任务 | ||
| 80 | - this_task.update({"state": 1,"process":"入库完成","update_time": datetime.datetime.now()}) | ||
| 81 | - this_task.commit() | ||
| 82 | - else: | ||
| 83 | - # 更新任务为失败任务 | ||
| 84 | - this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | ||
| 85 | - # rollback | ||
| 86 | - this_task.rollback() | ||
| 87 | - | ||
| 88 | - except Exception as e: | ||
| 89 | - this_task.write_process(e.__str__()) | ||
| 90 | - this_task.write_process("任务中止!") | ||
| 91 | - | ||
| 92 | - this_task.update({"state": -1, "process": "入库失败", "update_time": datetime.datetime.now()}) | ||
| 93 | - StructurePrint().print(e.__str__(),"ERROR") | ||
| 94 | - # rollback | ||
| 95 | - this_task.rollback() | ||
| 96 | - finally: | ||
| 97 | - this_task.end() | ||
| 98 | - try: | ||
| 99 | - file_tmp_path = os.path.join(_data_path.split("file_tmp")[0],"file_tmp") | ||
| 100 | - dir_path = os.path.dirname(_data_path) | ||
| 101 | - i=0 | ||
| 102 | - while not os.path.dirname(dir_path).__eq__(file_tmp_path) and i<30: | ||
| 103 | - dir_path = os.path.dirname(dir_path) | ||
| 104 | - i+=1 | ||
| 105 | - if i<30: | ||
| 106 | - shutil.rmtree(dir_path,True) | ||
| 107 | - StructurePrint().print("删除文件成功!") | ||
| 108 | - else: | ||
| 109 | - raise Exception("找不到文件!") | ||
| 110 | - | ||
| 111 | - except Exception as e: | ||
| 112 | - StructurePrint().print(e.__str__(), "ERROR") | ||
| 113 | - StructurePrint().print("删除文件失败!","ERROR") | ||
| 114 | - | ||
| 115 | - | ||
| 116 | - def entry_shp(self,data_path,this_task,meta): | ||
| 117 | - ''' | ||
| 118 | - 录入shp | ||
| 119 | - :param data_path: | ||
| 120 | - :return: | ||
| 121 | - ''' | ||
| 122 | - | ||
| 123 | - driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | ||
| 124 | - ds: DataSource = driver.Open(data_path, 1) | ||
| 125 | - if not ds: | ||
| 126 | - raise Exception("打开数据失败!") | ||
| 127 | - layer: Layer = ds.GetLayer(0) | ||
| 128 | - | ||
| 129 | - is_success_one, new_layer_name =self.entry_one_layer(layer, this_task,meta) | ||
| 130 | - ds.Destroy() | ||
| 131 | - return is_success_one, new_layer_name | ||
| 132 | - | ||
| 133 | - def entry_gdb(self,data_path,this_task,meta): | ||
| 134 | - ''' | ||
| 135 | - 录入gdb | ||
| 136 | - :param data_path: | ||
| 137 | - :return: | ||
| 138 | - ''' | ||
| 139 | - | ||
| 140 | - is_successes = [] | ||
| 141 | - new_layer_names=[] | ||
| 142 | - driver: Driver = ogr.GetDriverByName("OpenFileGDB") | ||
| 143 | - ds: DataSource = driver.Open(data_path, 0) | ||
| 144 | - if not ds: | ||
| 145 | - raise Exception("打开数据失败!") | ||
| 146 | - | ||
| 147 | - | ||
| 148 | - for i in range(ds.GetLayerCount()): | ||
| 149 | - | ||
| 150 | - layer: Layer = ds.GetLayer(i) | ||
| 151 | - if layer.GetName().lower() not in meta.get("layer").keys(): | ||
| 152 | - continue | ||
| 153 | - is_success, new_layer_name = self.entry_one_layer(layer,this_task,meta) | ||
| 154 | - new_layer_names.append(new_layer_name) | ||
| 155 | - is_successes.append(is_success) | ||
| 156 | - ds.Destroy() | ||
| 157 | - if is_successes.__contains__(False): | ||
| 158 | - return False,new_layer_names | ||
| 159 | - else: | ||
| 160 | - return True,new_layer_names | ||
| 161 | - | ||
| 162 | - def entry_one_layer(self,layer: Layer,this_task,meta): | ||
| 163 | - | ||
| 164 | - # this_task.pg_ds.StartTransaction() | ||
| 165 | - new_layer_name = None | ||
| 166 | - # vacuate_process= None | ||
| 167 | - success = True | ||
| 168 | - table_guid = uuid.uuid1().__str__() | ||
| 169 | - try: | ||
| 170 | - # 图层设置 | ||
| 171 | - parameter = this_task.parameter | ||
| 172 | - | ||
| 173 | - overwrite = parameter.get("overwrite") if parameter.get("overwrite") is not None and parameter.get("overwrite")=="yes" else "no" | ||
| 174 | - geom_name = parameter.get("geom_name") if parameter.get("geom_name") is not None else "geom" | ||
| 175 | - fid = parameter.get("fid") if parameter.get("fid") is not None else "fid" | ||
| 176 | - options = ["OVERWRITE={}".format(overwrite), "FID={}".format(fid), "GEOMETRY_NAME={}".format(geom_name),"PRECISION=NO"] | ||
| 177 | - | ||
| 178 | - | ||
| 179 | - # 将线/面转多线多面 | ||
| 180 | - geom_type = GeometryAdapter.change_geom_type(layer.GetGeomType()) | ||
| 181 | - | ||
| 182 | - # 更改图层名 | ||
| 183 | - change_name = False | ||
| 184 | - origin_name = layer.GetName().lower() | ||
| 185 | - | ||
| 186 | - # 新图层名 | ||
| 187 | - new_layer_name: str = meta.get("layer").get(origin_name) | ||
| 188 | - origin_name = new_layer_name | ||
| 189 | - no = 1 | ||
| 190 | - while overwrite.__eq__("no") and this_task.pg_ds.GetLayerByName(new_layer_name) : | ||
| 191 | - change_name=True | ||
| 192 | - new_layer_name = origin_name+"_{}".format(no) | ||
| 193 | - no+=1 | ||
| 194 | - | ||
| 195 | - if change_name: | ||
| 196 | - this_task.write_process("{}图层已存在,更名为{}入库".format(origin_name, new_layer_name)) | ||
| 197 | - | ||
| 198 | - | ||
| 199 | - this_task.write_process("{}图层正在入库...".format(new_layer_name)) | ||
| 200 | - | ||
| 201 | - pg_layer: Layer = this_task.pg_ds.CreateLayer(new_layer_name, layer.GetSpatialRef(), geom_type, options) | ||
| 202 | - | ||
| 203 | - # 复制原图层的属性 | ||
| 204 | - # 去掉fid的属性 | ||
| 205 | - schema = [sche for sche in layer.schema if not sche.name.__eq__(fid)] | ||
| 206 | - pg_layer.CreateFields(schema) | ||
| 207 | - | ||
| 208 | - #创建抽稀过程 | ||
| 209 | - # vacuate_process = VacuateProcess(layer,table_guid,options) | ||
| 210 | - | ||
| 211 | - | ||
| 212 | - | ||
| 213 | - count =0 | ||
| 214 | - | ||
| 215 | - for feature in layer: | ||
| 216 | - count+=1 | ||
| 217 | - if count%10000==0: | ||
| 218 | - StructurePrint().print("{}图层已入库{}个对象".format(new_layer_name,count)) | ||
| 219 | - # print(time.time()-this_time) | ||
| 220 | - #this_time=time.time() | ||
| 221 | - geo :Geometry = feature.GetGeometryRef() | ||
| 222 | - # 如果是空对象不录入 | ||
| 223 | - if geo is not None: | ||
| 224 | - if geo.IsEmpty(): | ||
| 225 | - this_task.write_process("FID:{}要素的空间字段为空,跳过该要素!".format(feature.GetFID())) | ||
| 226 | - StructurePrint().print("FID:{}要素的空间字段为空,跳过该要素!".format(feature.GetFID()),"WARN") | ||
| 227 | - continue | ||
| 228 | - | ||
| 229 | - out_feature: Feature = copy.copy(feature) | ||
| 230 | - out_geom = None | ||
| 231 | - if geo is not None: | ||
| 232 | - out_geom:Geometry = GeometryAdapter.change_geom(geo, geom_type) | ||
| 233 | - out_feature.SetGeometry(out_geom) | ||
| 234 | - # 出现fid为0经常有问题 | ||
| 235 | - out_feature.SetFID(out_feature.GetFID() + 1) | ||
| 236 | - pg_layer.CreateFeature(out_feature) | ||
| 237 | - | ||
| 238 | - #插入抽稀图层 | ||
| 239 | - # if out_geom is not None: | ||
| 240 | - # vacuate_process.vacuate(out_geom) | ||
| 241 | - | ||
| 242 | - # 注册图层信息 | ||
| 243 | - # 是否抽吸过 | ||
| 244 | - # is_vacuate = 1 if vacuate_process.max_level>0 else 0 | ||
| 245 | - is_vacuate = 0 | ||
| 246 | - | ||
| 247 | - this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | ||
| 248 | - | ||
| 249 | - # 注册抽稀表 | ||
| 250 | - # this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) | ||
| 251 | - | ||
| 252 | - this_task.write_process("{}图层入库成功。".format(new_layer_name)) | ||
| 253 | - | ||
| 254 | - except Exception as e: | ||
| 255 | - | ||
| 256 | - this_task.write_process("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__())) | ||
| 257 | - StructurePrint().print("{}入库失败,数据回滚!原因:{}".format(new_layer_name,e.__str__()), "error") | ||
| 258 | - print(traceback.format_exc()) | ||
| 259 | - # 抽稀回滚 | ||
| 260 | - # vacuate_process.rollback() | ||
| 261 | - success =False | ||
| 262 | - | ||
| 263 | - finally: | ||
| 264 | - # vacuate_process.end() | ||
| 265 | - pass | ||
| 266 | - return success,new_layer_name | ||
| 267 | - | ||
| 268 | - | ||
| 269 | - | ||
| 270 | -class ThisTask: | ||
| 271 | - | ||
| 272 | - def __init__(self, parameter): | ||
| 273 | - try: | ||
| 274 | - # 该任务事务的连接 | ||
| 275 | - self.sys_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
| 276 | - # 专门的写过程的连接 | ||
| 277 | - self.process_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
| 278 | - | ||
| 279 | - except Exception as e: | ||
| 280 | - raise Exception("打开数据库失败!") | ||
| 281 | - self.parameter = parameter | ||
| 282 | - | ||
| 283 | - self.task = self.process_session.query(Task).filter_by(guid=parameter.get("task_guid")) | ||
| 284 | - | ||
| 285 | - self.database = self.sys_session.query(Database).filter_by( | ||
| 286 | - guid=parameter.get("database_guid")).one_or_none() | ||
| 287 | - | ||
| 288 | - self.catalog_guid = parameter.get("catalog_guid") | ||
| 289 | - | ||
| 290 | - self.pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(self.database.sqlalchemy_uri)) | ||
| 291 | - | ||
| 292 | - | ||
| 293 | - def start(self): | ||
| 294 | - self.pg_ds.StartTransaction() | ||
| 295 | - | ||
| 296 | - def update(self, update_dict): | ||
| 297 | - self.task.update(update_dict) | ||
| 298 | - self.process_session.commit() | ||
| 299 | - | ||
| 300 | - def write_process(self, message): | ||
| 301 | - message = "{} {}".format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), message) | ||
| 302 | - task_process_guid = uuid.uuid1().__str__() | ||
| 303 | - task_process = Process(guid=task_process_guid, message=message, time=datetime.datetime.now(), | ||
| 304 | - task_guid=self.parameter.get("task_guid")) | ||
| 305 | - self.process_session.add(task_process) | ||
| 306 | - self.process_session.commit() | ||
| 307 | - | ||
| 308 | - def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate,table_guid): | ||
| 309 | - ''' | ||
| 310 | - 注册表 | ||
| 311 | - :param layer: 图层 | ||
| 312 | - :param new_layer_name: 图层名 | ||
| 313 | - :return: 表名 | ||
| 314 | - ''' | ||
| 315 | - | ||
| 316 | - this_time = datetime.datetime.now() | ||
| 317 | - | ||
| 318 | - ext = layer.GetExtent() | ||
| 319 | - if ext[0] < 360: | ||
| 320 | - ext = [round(e, 6) for e in ext] | ||
| 321 | - else: | ||
| 322 | - ext = [round(e, 2) for e in ext] | ||
| 323 | - | ||
| 324 | - geom_type = GeometryAdapter.get_geometry_type(layer) | ||
| 325 | - | ||
| 326 | - extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) | ||
| 327 | - | ||
| 328 | - table = Table(guid=table_guid, | ||
| 329 | - database_guid=self.database.guid, | ||
| 330 | - creator=creator, | ||
| 331 | - name=new_layer_name, create_time=this_time, update_time=this_time, | ||
| 332 | - catalog_guid=self.catalog_guid, table_type=GeometryAdapter.get_table_type(geom_type), | ||
| 333 | - extent=extent, | ||
| 334 | - feature_count=layer.GetFeatureCount(), | ||
| 335 | - is_vacuate=is_vacuate | ||
| 336 | - ) | ||
| 337 | - # 删除遗留业务数据 | ||
| 338 | - try: | ||
| 339 | - history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all() | ||
| 340 | - except: | ||
| 341 | - self.sys_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) | ||
| 342 | - history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all() | ||
| 343 | - | ||
| 344 | - if history_table: | ||
| 345 | - for ht in history_table: | ||
| 346 | - self.sys_session.delete(ht) | ||
| 347 | - self.sys_session.add(table) | ||
| 348 | - | ||
| 349 | - feature_defn: FeatureDefn = layer.GetLayerDefn() | ||
| 350 | - | ||
| 351 | - for i in range(feature_defn.GetFieldCount()): | ||
| 352 | - field_defn: FieldDefn = feature_defn.GetFieldDefn(i) | ||
| 353 | - field_name = field_defn.GetName().lower() | ||
| 354 | - field_alias = field_name if field_defn.GetAlternativeName() is None or field_defn.GetAlternativeName().__eq__( | ||
| 355 | - "") else field_defn.GetAlternativeName() | ||
| 356 | - column = Columns(guid=uuid.uuid1().__str__(), table_guid=table_guid, | ||
| 357 | - name=field_name, alias=field_alias, create_time=this_time, update_time=this_time) | ||
| 358 | - self.sys_session.add(column) | ||
| 359 | - return table_guid | ||
| 360 | - | ||
| 361 | - | ||
| 362 | - def register_table_vacuate(self,table_guid,vacuate_layers:dict): | ||
| 363 | - | ||
| 364 | - # 抽稀表有固定的命名规则 | ||
| 365 | - for level,layer in vacuate_layers.items(): | ||
| 366 | - pixel_distance_str:str=layer.GetName().split("_")[-1] | ||
| 367 | - lev = layer.GetName().split("_")[-2] | ||
| 368 | - | ||
| 369 | - if pixel_distance_str.startswith("0"): | ||
| 370 | - pixel_distance_str="0.{}".format(pixel_distance_str) | ||
| 371 | - | ||
| 372 | - pixel_distance = float(pixel_distance_str) | ||
| 373 | - | ||
| 374 | - table_vacuate = TableVacuate(guid=uuid.uuid1().__str__(), | ||
| 375 | - table_guid=table_guid, | ||
| 376 | - level=int(lev), | ||
| 377 | - name=layer.GetName(), | ||
| 378 | - pixel_distance=pixel_distance) | ||
| 379 | - self.sys_session.add(table_vacuate) | ||
| 380 | - | ||
| 381 | - | ||
| 382 | - def commit(self): | ||
| 383 | - if self.sys_session: | ||
| 384 | - self.sys_session.commit() | ||
| 385 | - if self.pg_ds: | ||
| 386 | - self.pg_ds.CommitTransaction() | ||
| 387 | - if self.process_session: | ||
| 388 | - self.process_session.commit() | ||
| 389 | - | ||
| 390 | - | ||
| 391 | - def end(self): | ||
| 392 | - if self.sys_session: | ||
| 393 | - self.sys_session.close() | ||
| 394 | - if self.pg_ds: | ||
| 395 | - self.pg_ds.Destroy() | ||
| 396 | - if self.process_session: | ||
| 397 | - self.process_session.close() | ||
| 398 | - | ||
| 399 | - def rollback(self): | ||
| 400 | - if self.sys_session: | ||
| 401 | - self.sys_session.rollback() | ||
| 402 | - if self.pg_ds: | ||
| 403 | - self.pg_ds.RollbackTransaction() | ||
| 404 | - | ||
| 405 | - | ||
| 406 | - | ||
| 407 | -class VacuateProcess: | ||
| 408 | - | ||
| 409 | - max_level=0 | ||
| 410 | - fill_dict={} | ||
| 411 | - vacuate_layers={} | ||
| 412 | - vacuate_layers_gridsize={} | ||
| 413 | - pg_ds_dict = {} | ||
| 414 | - # 图层要素大于5W才抽稀 | ||
| 415 | - least_vacuate_count = VacuateConf.least_vacuate_count | ||
| 416 | - | ||
| 417 | - extent=[] | ||
| 418 | - is_spatial=False | ||
| 419 | - | ||
| 420 | - lonlat_gridsize = VacuateConf.lonlat_gridsize | ||
| 421 | - project_gridsize = VacuateConf.project_gridsize | ||
| 422 | - | ||
| 423 | - # 该抽稀过程使用的grid_size | ||
| 424 | - t_grid_size = [] | ||
| 425 | - | ||
| 426 | - # 该抽稀过程的抽稀网格 | ||
| 427 | - this_gridsize=[] | ||
| 428 | - | ||
| 429 | - | ||
| 430 | - def __init__(self,layer:Layer,table_guid, options,sqlalchemy_uri): | ||
| 431 | - | ||
| 432 | - #是空间图层才初始化 | ||
| 433 | - if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: | ||
| 434 | - | ||
| 435 | - self.is_spatial=True | ||
| 436 | - | ||
| 437 | - # 判断需要抽稀多少级 | ||
| 438 | - | ||
| 439 | - lc = layer.GetFeatureCount() | ||
| 440 | - extent = layer.GetExtent() | ||
| 441 | - self.extent=extent | ||
| 442 | - | ||
| 443 | - #判断疏密程度 | ||
| 444 | - p_x = (extent[1]-extent[0])/10.0 | ||
| 445 | - p_y = (extent[3] - extent[2]) / 10.0 | ||
| 446 | - fill_precent=0 | ||
| 447 | - StructurePrint().print("判断疏密") | ||
| 448 | - for ix in range(10): | ||
| 449 | - for iy in range(10): | ||
| 450 | - grid_extent = [extent[0]+ix*p_x,extent[0]+ix*p_x+p_x,extent[2]+iy*p_y,extent[2]+iy*p_y+p_y] | ||
| 451 | - poly = GeometryAdapter.envelop_2_polygon(grid_extent) | ||
| 452 | - | ||
| 453 | - layer.SetSpatialFilter(None) | ||
| 454 | - layer.SetSpatialFilter(poly) | ||
| 455 | - layer.ResetReading() | ||
| 456 | - if layer.GetNextFeature(): | ||
| 457 | - fill_precent += 1 | ||
| 458 | - | ||
| 459 | - print(fill_precent) | ||
| 460 | - StructurePrint().print("判断疏密结束") | ||
| 461 | - | ||
| 462 | - layer.SetSpatialFilter(None) | ||
| 463 | - layer.ResetReading() | ||
| 464 | - # 固有疏密程度 | ||
| 465 | - original_density=8 | ||
| 466 | - | ||
| 467 | - | ||
| 468 | - # 额外一层 | ||
| 469 | - # self.this_gridsize.append(0.000075) | ||
| 470 | - # self.max_level += 1 | ||
| 471 | - ###### | ||
| 472 | - | ||
| 473 | - if extent[0]>180: | ||
| 474 | - self.t_grid_size=self.project_gridsize | ||
| 475 | - else: | ||
| 476 | - self.t_grid_size = self.lonlat_gridsize | ||
| 477 | - | ||
| 478 | - for grid_size in self.t_grid_size: | ||
| 479 | - # 最少抽稀个数 | ||
| 480 | - if lc > self.least_vacuate_count: | ||
| 481 | - # 网格数至少大于 | ||
| 482 | - if ((extent[1] - extent[0]) * (extent[3] - extent[2])) / (grid_size**2)>self.least_vacuate_count: | ||
| 483 | - # 要素数量大于网格数量 | ||
| 484 | - # 要考虑图层的疏密程度,original_density*(100.0/fill_precent) 为疏密指数 | ||
| 485 | - if lc * original_density * (100.0/fill_precent)>((extent[1] - extent[0])*(extent[3] - extent[2]))/(grid_size**2) : | ||
| 486 | - print(grid_size) | ||
| 487 | - self.this_gridsize.append(grid_size) | ||
| 488 | - self.max_level += 1 | ||
| 489 | - | ||
| 490 | - | ||
| 491 | - | ||
| 492 | - # 创建抽稀ds | ||
| 493 | - for l in range(self.max_level): | ||
| 494 | - # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | ||
| 495 | - if configure.VACUATE_DB_URI: | ||
| 496 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) | ||
| 497 | - else: | ||
| 498 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | ||
| 499 | - pg_ds_l.StartTransaction() | ||
| 500 | - self.pg_ds_dict[l] = pg_ds_l | ||
| 501 | - | ||
| 502 | - # 生成抽稀图层 | ||
| 503 | - options = options[1:] | ||
| 504 | - options.append("OVERWRITE=yes") | ||
| 505 | - options.append("LAUNDER=no") | ||
| 506 | - | ||
| 507 | - schema = layer.schema | ||
| 508 | - # 增加统计字段 | ||
| 509 | - schema.append(ogr.FieldDefn("_dcigrid_count_", ogr.OFTInteger)) | ||
| 510 | - schema.append(ogr.FieldDefn("_dcigrid_name_", ogr.OFTString)) | ||
| 511 | - | ||
| 512 | - for l in range(self.max_level): | ||
| 513 | - this_grid_len = self.this_gridsize[l] | ||
| 514 | - | ||
| 515 | - self.vacuate_layers_gridsize[l] = this_grid_len | ||
| 516 | - | ||
| 517 | - pg = self.pg_ds_dict[l] | ||
| 518 | - | ||
| 519 | - grid_name = str(this_grid_len) | ||
| 520 | - if this_grid_len<1: | ||
| 521 | - grid_name = str(this_grid_len).split(".")[-1] | ||
| 522 | - if this_grid_len.__eq__(0.00008): | ||
| 523 | - grid_name = "00008" | ||
| 524 | - | ||
| 525 | - # 抽稀图层是点面混合的 | ||
| 526 | - # 抽稀表有固定的命名规则 | ||
| 527 | - # 抽稀表一定要覆盖 | ||
| 528 | - | ||
| 529 | - | ||
| 530 | - print("{}:{}".format(self.t_grid_size.index(this_grid_len),this_grid_len)) | ||
| 531 | - | ||
| 532 | - | ||
| 533 | - v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name) | ||
| 534 | - vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | ||
| 535 | - # 抽稀表需要属性 | ||
| 536 | - vl.CreateFields(schema) | ||
| 537 | - self.vacuate_layers[l] = vl | ||
| 538 | - | ||
| 539 | - else: | ||
| 540 | - pass | ||
| 541 | - | ||
| 542 | - | ||
| 543 | - def vacuate(self,g,feature): | ||
| 544 | - | ||
| 545 | - if self.is_spatial: | ||
| 546 | - | ||
| 547 | - # 插入到所有抽稀图层中 | ||
| 548 | - for level in range(self.max_level): | ||
| 549 | - | ||
| 550 | - center: Geometry = g.Centroid() | ||
| 551 | - | ||
| 552 | - extent = g.GetEnvelope() | ||
| 553 | - long_extent= extent[1]-extent[0] | ||
| 554 | - lat_extent = extent[3]-extent[2] | ||
| 555 | - | ||
| 556 | - this_grid_len =self.vacuate_layers_gridsize[level] | ||
| 557 | - #超大的直接加入 | ||
| 558 | - # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | ||
| 559 | - # vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
| 560 | - # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
| 561 | - # feat.SetGeometry(g) | ||
| 562 | - # vacuate_layer.CreateFeature(feat) | ||
| 563 | - # else: | ||
| 564 | - | ||
| 565 | - row = int((center.GetY() - self.extent[2]) / this_grid_len) | ||
| 566 | - col = int((center.GetX() - self.extent[0]) / this_grid_len) | ||
| 567 | - key = "{}.{}.{}".format(level, row, col) | ||
| 568 | - | ||
| 569 | - if not self.fill_dict.get(key): | ||
| 570 | - self.fill_dict[key] = 0 | ||
| 571 | - if self.fill_dict[key] == 0: | ||
| 572 | - | ||
| 573 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
| 574 | - feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
| 575 | - # 如果图形比网格小,直接存储其中心点 | ||
| 576 | - if this_grid_len>long_extent and this_grid_len>lat_extent: | ||
| 577 | - feat.SetGeometry(center) | ||
| 578 | - else: | ||
| 579 | - feat.SetGeometry(g) | ||
| 580 | - | ||
| 581 | - # 复制旧feature属性 | ||
| 582 | - field_dict = feature.items() | ||
| 583 | - for field_name in field_dict: | ||
| 584 | - feat.SetField(field_name, field_dict[field_name]) | ||
| 585 | - feat.SetField("_dcigrid_name_",".".join(key.split(".")[1:])) | ||
| 586 | - | ||
| 587 | - vacuate_layer.CreateFeature(feat) | ||
| 588 | - self.fill_dict[key] += 1 | ||
| 589 | - #超大的还有机会 | ||
| 590 | - elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | ||
| 591 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
| 592 | - feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | ||
| 593 | - feat.SetGeometry(g) | ||
| 594 | - | ||
| 595 | - # 复制旧feature属性 | ||
| 596 | - field_dict = feature.items() | ||
| 597 | - for field_name in field_dict: | ||
| 598 | - feat.SetField(field_name, field_dict[field_name]) | ||
| 599 | - feat.SetField("_dcigrid_name_",".".join(key.split(".")[1:])) | ||
| 600 | - | ||
| 601 | - vacuate_layer.CreateFeature(feat) | ||
| 602 | - self.fill_dict[key] += 1 | ||
| 603 | - else: | ||
| 604 | - self.fill_dict[key] += 1 | ||
| 605 | - | ||
| 606 | - def set_vacuate_count(self): | ||
| 607 | - if self.is_spatial: | ||
| 608 | - # 插入到所有抽稀图层中 | ||
| 609 | - for level in range(self.max_level): | ||
| 610 | - vacuate_layer: Layer = self.vacuate_layers.get(level) | ||
| 611 | - for feat in vacuate_layer: | ||
| 612 | - key = "{}.{}".format(level,feat.GetField("_dcigrid_name_")) | ||
| 613 | - feat.SetField("_dcigrid_count_",self.fill_dict.get(key)) | ||
| 614 | - vacuate_layer.SetFeature(feat) | ||
| 615 | - | ||
| 616 | - def end(self): | ||
| 617 | - for pg in self.pg_ds_dict.values(): | ||
| 618 | - pg.Destroy() | ||
| 619 | - | ||
| 620 | - def rollback(self): | ||
| 621 | - for pg in self.pg_ds_dict.values(): | ||
| 622 | - pg.RollbackTransaction() |
| @@ -6,8 +6,9 @@ | @@ -6,8 +6,9 @@ | ||
| 6 | from flask import request | 6 | from flask import request |
| 7 | import uuid | 7 | import uuid |
| 8 | import os | 8 | import os |
| 9 | - | ||
| 10 | - | 9 | +import stat |
| 10 | +import platform | ||
| 11 | +from app.util.component.StructuredPrint import StructurePrint | ||
| 11 | class FileProcess: | 12 | class FileProcess: |
| 12 | @classmethod | 13 | @classmethod |
| 13 | def save(cls,parent): | 14 | def save(cls,parent): |
| @@ -21,9 +22,15 @@ class FileProcess: | @@ -21,9 +22,15 @@ class FileProcess: | ||
| 21 | os.makedirs(dir_path) | 22 | os.makedirs(dir_path) |
| 22 | #有时候文件最后会多一个" | 23 | #有时候文件最后会多一个" |
| 23 | filename = file.filename.split('"')[0] | 24 | filename = file.filename.split('"')[0] |
| 25 | + | ||
| 24 | store_file = os.path.join(dir_path, filename) | 26 | store_file = os.path.join(dir_path, filename) |
| 27 | + StructurePrint().print(store_file) | ||
| 28 | + | ||
| 25 | file.save(store_file) | 29 | file.save(store_file) |
| 26 | 30 | ||
| 31 | + if platform.system().lower() == 'linux': | ||
| 32 | + #设置为444权限 | ||
| 33 | + os.chmod(store_file, stat.S_IRUSR + stat.S_IRGRP + stat.S_IROTH) | ||
| 27 | return dir_path, store_file | 34 | return dir_path, store_file |
| 28 | 35 | ||
| 29 | @classmethod | 36 | @classmethod |
| @@ -36,6 +36,7 @@ class ZipUtil: | @@ -36,6 +36,7 @@ class ZipUtil: | ||
| 36 | name_t = names.encode('cp437').decode('gbk') | 36 | name_t = names.encode('cp437').decode('gbk') |
| 37 | except: | 37 | except: |
| 38 | name_t = names.encode('utf-8').decode('utf-8') | 38 | name_t = names.encode('utf-8').decode('utf-8') |
| 39 | + # name_t = names | ||
| 39 | 40 | ||
| 40 | zip_file.extract(names,store_path) | 41 | zip_file.extract(names,store_path) |
| 41 | 42 |
envvars
0 → 100644
| 1 | +# envvars - default environment variables for apache2ctl | ||
| 2 | + | ||
| 3 | +# this won't be correct after changing uid | ||
| 4 | +unset HOME | ||
| 5 | + | ||
| 6 | +# for supporting multiple apache2 instances | ||
| 7 | +if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then | ||
| 8 | + SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}" | ||
| 9 | +else | ||
| 10 | + SUFFIX= | ||
| 11 | +fi | ||
| 12 | + | ||
| 13 | +# Since there is no sane way to get the parsed apache2 config in scripts, some | ||
| 14 | +# settings are defined via environment variables and then used in apache2ctl, | ||
| 15 | +# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc. | ||
| 16 | +export APACHE_RUN_USER=www-data | ||
| 17 | +export APACHE_RUN_GROUP=www-data | ||
| 18 | +# temporary state file location. This might be changed to /run in Wheezy+1 | ||
| 19 | +export APACHE_PID_FILE=/var/run/apache2$SUFFIX/apache2.pid | ||
| 20 | +export APACHE_RUN_DIR=/var/run/apache2$SUFFIX | ||
| 21 | +export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX | ||
| 22 | +# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. | ||
| 23 | +export APACHE_LOG_DIR=/var/log/apache2$SUFFIX | ||
| 24 | + | ||
| 25 | +## The locale used by some modules like mod_dav | ||
| 26 | +export LANG="C.UTF-8" | ||
| 27 | +## Uncomment the following line to use the system default locale instead: | ||
| 28 | +export LANG | ||
| 29 | + | ||
| 30 | +export TZ="Asia/Shanghai" | ||
| 31 | + | ||
| 32 | +## The command to get the status for 'apache2ctl status'. | ||
| 33 | +## Some packages providing 'www-browser' need '--dump' instead of '-dump'. | ||
| 34 | +#export APACHE_LYNX='www-browser -dump' | ||
| 35 | + | ||
| 36 | +## If you need a higher file descriptor limit, uncomment and adjust the | ||
| 37 | +## following line (default is 8192): | ||
| 38 | +#APACHE_ULIMIT_MAX_FILES='ulimit -n 65536' | ||
| 39 | + | ||
| 40 | +## If you would like to pass arguments to the web server, add them below | ||
| 41 | +## to the APACHE_ARGUMENTS environment. | ||
| 42 | +#export APACHE_ARGUMENTS='' | ||
| 43 | + | ||
| 44 | +## Enable the debug mode for maintainer scripts. | ||
| 45 | +## This will produce a verbose output on package installations of web server modules and web application | ||
| 46 | +## installations which interact with Apache | ||
| 47 | +#export APACHE2_MAINTSCRIPT_DEBUG=1 |
| 1 | -flask==1.1.2 | ||
| 2 | -SQLAlchemy==1.3.17 | ||
| 3 | -Flask-SQLAlchemy==2.4.3 | ||
| 4 | -gevent==20.9.0 | ||
| 5 | -gunicorn==20.0.4 | ||
| 6 | -flask_cors==3.0.8 | ||
| 7 | -flasgger==0.9.5 | ||
| 8 | -#GDAL==3.2.1 | ||
| 9 | -psycopg2-binary==2.8.5 | ||
| 10 | -pyDes==2.0.1 | ||
| 11 | -gevent-websocket==0.10.1 | ||
| 12 | -opencv-python==4.5.1.48 | ||
| 13 | -mod_wsgi==4.8.0 | ||
| 14 | -thrift==0.13.0 | ||
| 15 | -Authlib==0.13 | ||
| 16 | -kazoo==2.8.0 | ||
| 17 | -paramiko==2.8.0 | ||
| 18 | -requests==2.26.0 | ||
| 19 | -schedule==1.1.0 | 1 | +flask==1.1.2 |
| 2 | +SQLAlchemy==1.3.17 | ||
| 3 | +Flask-SQLAlchemy==2.4.3 | ||
| 4 | +gevent==20.9.0 | ||
| 5 | +gunicorn==20.0.4 | ||
| 6 | +flask_cors==3.0.8 | ||
| 7 | +flasgger==0.9.5 | ||
| 8 | +#GDAL==3.2.1 | ||
| 9 | +psycopg2-binary==2.8.5 | ||
| 10 | +pyDes==2.0.1 | ||
| 11 | +gevent-websocket==0.10.1 | ||
| 12 | +opencv-python==4.5.1.48 | ||
| 13 | +mod_wsgi==4.8.0 | ||
| 14 | +thrift==0.13.0 | ||
| 15 | +Authlib==0.13 | ||
| 16 | +kazoo==2.8.0 | ||
| 17 | +paramiko==2.8.0 | ||
| 18 | +requests==2.26.0 | ||
| 19 | +schedule==1.1.0 |
| @@ -16,7 +16,7 @@ else | @@ -16,7 +16,7 @@ else | ||
| 16 | echo "端口设置为$1 ..." | 16 | echo "端口设置为$1 ..." |
| 17 | fi | 17 | fi |
| 18 | 18 | ||
| 19 | -docker run -d --name $dn -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -p $port:8840 -v $curPath:/usr/src/app -w /usr/src/app dci/dmapdms:3.0 python3 ./run.py | 19 | +docker run -d --name $dn -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -p $port:8840 -v $curPath:/usr/src/app -w /usr/src/app dci/dmapserver:build python3 ./run.py |
| 20 | # 清除未完成的任务任务 | 20 | # 清除未完成的任务任务 |
| 21 | sleep 5 | 21 | sleep 5 |
| 22 | curl localhost:$port/release | 22 | curl localhost:$port/release |
| @@ -33,8 +33,8 @@ fi | @@ -33,8 +33,8 @@ fi | ||
| 33 | 33 | ||
| 34 | #启动容器和apache | 34 | #启动容器和apache |
| 35 | echo "正在启动容器..." | 35 | echo "正在启动容器..." |
| 36 | -set="--privileged=true -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0" | ||
| 37 | -docker run -it -d --name $container_name $set -p ${port}:80 -p ${port2}:81 -v ${curPath}:/usr/src/app -v ${curPath}/apache2.conf:/etc/apache2/apache2.conf -v ${curPath}/dmapmanager.conf:/etc/apache2/sites-enabled/dmapmanager.conf dci/dmapserver:base | 36 | +set="--privileged=true -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0 -e LAND=C.UTF-8" |
| 37 | +docker run -it -d --name ${container_name} ${set} -p ${port}:80 -p ${port2}:81 -v ${curPath}:/usr/src/app -v ${curPath}/apache2.conf:/etc/apache2/apache2.conf -v ${curPath}/dmapmanager.conf:/etc/apache2/sites-enabled/dmapmanager.conf -v ${curPath}/envvars:/etc/apache2/envvars dci/dmapserver:build | ||
| 38 | docker exec -d $container_name service apache2 start | 38 | docker exec -d $container_name service apache2 start |
| 39 | sleep 5 | 39 | sleep 5 |
| 40 | curl localhost:$port/release | 40 | curl localhost:$port/release |
请
注册
或
登录
后发表评论