HWJ 何伟俊 / DMapManager · 提交

提交 afdd3afd321bbc502c16df504d0eb391b1520900

作者 nheweijun
1 个父辈 001fd3cb

2022.04.15 修复任务线程比创建任务更快BUG

内嵌 并排对比
正在显示 14 个修改的文件 包含 370 行增加 和 24 行删除
... ... @@ -33,8 +33,7 @@ class Api(ApiTemplate):
33 33 try:
34 34
35 35 task_guid = uuid.uuid1().__str__()
36   - download_process = multiprocessing.Process(target=self.download, args=(task_guid,self.para))
37   - download_process.start()
  36 +
38 37
39 38 task = Task(guid=task_guid,
40 39 name="下载 | {}".format(self.para.get("table_name")),
... ... @@ -45,11 +44,18 @@ class Api(ApiTemplate):
45 44 file_name=None,
46 45 process="数据下载中",
47 46 database_guid=self.para.get("database_guid"),
48   - task_pid=download_process.pid)
  47 + # task_pid=download_process.pid
  48 + )
49 49
50 50 db.session.add(task)
51 51 db.session.commit()
52 52
  53 + download_process = multiprocessing.Process(target=self.download, args=(task_guid,self.para))
  54 + download_process.start()
  55 +
  56 + task.task_pid = download_process.pid
  57 + db.session.commit()
  58 +
53 59 res["data"] = "下载任务已提交!"
54 60
55 61 # 提示信息
... ... @@ -79,8 +85,10 @@ class Api(ApiTemplate):
79 85 try:
80 86
81 87 #任务控制,等待执行
  88 +
82 89 TaskController.wait(task_guid)
83 90 task_writer = TaskWriter(task_guid)
  91 +
84 92 task_writer.update_task({"state":2,"update_time":datetime.datetime.now(),"process" : "下载中"})
85 93 task_writer.update_process("开始下载...")
86 94
... ... @@ -186,6 +194,9 @@ class Api(ApiTemplate):
186 194
187 195 # 创建一个gdb datasource
188 196 gdb_driver = ogr.GetDriverByName('FileGDB')
  197 + if not gdb_driver:
  198 + raise Exception("目前不支持GDB下载，驱动不存在！")
  199 +
189 200 uuid_ = uuid.uuid1().__str__()
190 201 parent = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
191 202
... ...
... ... @@ -76,8 +76,7 @@ class Api(ApiTemplate):
76 76 print("关闭数据库失败！")
77 77 return res
78 78
79   - entry_thread = multiprocessing.Process(target=self.entry,args=(self.para.get("task_guid"),))
80   - entry_thread.start()
  79 +
81 80
82 81
83 82 # 录入数据后台进程,录入主函数为entry
... ... @@ -93,11 +92,17 @@ class Api(ApiTemplate):
93 92 catalog_guid=self.para.get("catalog_guid"),
94 93 process="等待入库",
95 94 parameter=json.dumps(self.para),
96   - task_pid=entry_thread.pid)
  95 + # task_pid=entry_thread.pid
  96 + )
97 97 db.session.add(task)
98 98 db.session.commit()
99 99
100 100
  101 + entry_thread = multiprocessing.Process(target=self.entry,args=(self.para.get("task_guid"),))
  102 + entry_thread.start()
  103 + task.task_pid=entry_thread.pid
  104 + db.session.commit()
  105 +
101 106 res["result"] = True
102 107 res["msg"] = "数据录入提交成功！"
103 108 res["data"] = self.para["task_guid"]
... ... @@ -110,6 +115,7 @@ class Api(ApiTemplate):
110 115 task_writer = None
111 116 this_task_layer = []
112 117 try:
  118 +
113 119 #任务控制,等待执行
114 120 TaskController.wait(task_guid)
115 121
... ... @@ -160,6 +166,7 @@ class Api(ApiTemplate):
160 166 task_writer.session.delete(iln)
161 167
162 168 except Exception as e:
  169 + StructurePrint().print(e.__str__(), "error")
163 170 task_writer.update_task({"state": -1, "process": "入库失败"})
164 171 for ln in this_task_layer:
165 172 iln = task_writer.session.query(InsertingLayerName).filter_by(name=ln).one_or_none()
... ...
... ... @@ -35,9 +35,6 @@ class Api(ApiTemplate):
35 35 # 初始化task
36 36 task_guid = uuid.uuid1().__str__()
37 37
38   - refresh_process = multiprocessing.Process(target=self.table_refresh,args=(database,task_guid,self.para.get("creator")))
39   - refresh_process.start()
40   -
41 38 task = Task(guid=task_guid,
42 39 name="更新 | {}".format(database.alias),
43 40 create_time=datetime.datetime.now(),
... ... @@ -47,10 +44,17 @@ class Api(ApiTemplate):
47 44 file_name=None,
48 45 database_guid=database.guid,
49 46 process="数据库更新中",
50   - task_pid=refresh_process.pid)
  47 + # task_pid=refresh_process.pid
  48 + )
51 49
52 50 db.session.add(task)
53 51 db.session.commit()
  52 +
  53 + refresh_process = multiprocessing.Process(target=self.table_refresh,args=(database,task_guid,self.para.get("creator")))
  54 + refresh_process.start()
  55 + task.task_pid = refresh_process.pid
  56 + db.session.commit()
  57 +
54 58 res["msg"] = "数据库更新已提交！"
55 59 res["data"] = task_guid
56 60 res["result"] = True
... ...
... ... @@ -71,9 +71,6 @@ class Api(ApiTemplate):
71 71
72 72 # 初始化task
73 73 task_guid = uuid.uuid1().__str__()
74   - vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid))
75   - vacuate_process.start()
76   -
77 74
78 75 task = Task(guid=task_guid,
79 76 name="矢量金字塔 | {}".format(table.name),
... ... @@ -85,12 +82,19 @@ class Api(ApiTemplate):
85 82 file_name=None,
86 83 database_guid=table.database_guid,
87 84 process="构建中",
88   - task_pid= vacuate_process.pid
  85 + # task_pid= vacuate_process.pid
89 86 # parameter=",".join([str(x) for x in ref_grids])
90 87 )
91 88
92 89 db.session.add(task)
93 90 db.session.commit()
  91 +
  92 + vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid))
  93 + vacuate_process.start()
  94 +
  95 + task.task_pid = vacuate_process.pid
  96 + db.session.commit()
  97 +
94 98 res["msg"] = "矢量金字塔构建已提交！"
95 99 res["data"] = task_guid
96 100 res["result"] = True
... ...
... ... @@ -74,8 +74,7 @@ class Api(ApiTemplate):
74 74 # 初始化task
75 75 task_guid = uuid.uuid1().__str__()
76 76
77   - vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid,grids))
78   - vacuate_process.start()
  77 +
79 78
80 79
81 80 task = Task(guid=task_guid,
... ... @@ -90,10 +89,17 @@ class Api(ApiTemplate):
90 89 database_guid=table.database_guid,
91 90 process="构建中",
92 91 parameter=self.para.get("grids"),
93   - task_pid=vacuate_process.pid)
  92 + # task_pid=vacuate_process.pid
  93 + )
94 94
95 95 db.session.add(task)
96 96 db.session.commit()
  97 +
  98 + vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid,grids))
  99 + vacuate_process.start()
  100 + task.task_pid = vacuate_process.pid
  101 + db.session.commit()
  102 +
97 103 res["msg"] = "矢量金字塔构建已提交！"
98 104 res["data"] = task_guid
99 105 res["result"] = True
... ...
... ... @@ -25,13 +25,6 @@ class Api(ApiTemplate):
25 25 res = {}
26 26 try:
27 27 task_guid = uuid.uuid1().__str__()
28   -
29   -
30   - pyramid_process = multiprocessing.Process(target=self.task, args=(task_guid,self.para))
31   - pyramid_process.start()
32   -
33   -
34   -
35 28 task = Task(guid=task_guid,
36 29 name="影像金字塔 | {}".format(self.para.get("alias")),
37 30 create_time=datetime.datetime.now(),
... ... @@ -44,6 +37,9 @@ class Api(ApiTemplate):
44 37 db.session.add(task)
45 38 db.session.commit()
46 39
  40 + pyramid_process = multiprocessing.Process(target=self.task, args=(task_guid,self.para))
  41 + pyramid_process.start()
  42 +
47 43 res["data"] = "创建影像金字塔任务已提交！"
48 44 res["result"] = True
49 45
... ...
... ... @@ -31,6 +31,7 @@ class Api(ApiTemplate):
31 31 # 调用电子地图的注册服务接口
32 32
33 33 project_file = ProjectFile.create(self.para)
  34 +
34 35 para = {"name":self.para.get("name"),
35 36 "title":self.para.get("title"),
36 37 "type":"tileserver",
... ... @@ -41,6 +42,7 @@ class Api(ApiTemplate):
41 42 resp: Response = requests.post(tile_service_register_url,data=json.dumps(para),
42 43 headers={'Content-Type':'application/json'},timeout=3
43 44 )
  45 +
44 46 if resp.status_code == 200:
45 47 resp.encoding="utf-8"
46 48 resp_json = resp.json()
... ... @@ -143,6 +145,12 @@ class Api(ApiTemplate):
143 145 "in": "formData",
144 146 "type": "string",
145 147 "description": "[电子地图，影像地图]切片方案"},
  148 +
  149 + {"name": "scheme",
  150 + "in": "formData",
  151 + "type": "string",
  152 + "description": "[影像地图,电子地图]切片方案json"},
  153 +
146 154 {"name": "overview",
147 155 "in": "formData",
148 156 "type": "string",
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/12/6
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +from app.modules.service.models import TileScheme
  8 +from app.util.component.ModelVisitor import ModelVisitor
  9 +from app.util.component.StructurePrint import StructurePrint
  10 +import json
  11 +import base64
  12 +
  13 +
  14 +class ProjectFile:
  15 +
  16 + @classmethod
  17 + def create(cls,para):
  18 + if para.get("tile_type").__eq__("WMTS"):
  19 +
  20 + if para.get("scheme"):
  21 + tile_scheme = json.loads(para.get("scheme"))
  22 + else:
  23 + tile_scheme: TileScheme = TileScheme.query.filter_by(guid=para.get("scheme_guid")).one_or_none()
  24 + if not tile_scheme:
  25 + raise Exception("切片方案不存在！")
  26 + tile_scheme = ModelVisitor.object_to_json(tile_scheme)
  27 +
  28 + default_scheme = {}
  29 + other_scheme = []
  30 + if isinstance(tile_scheme,list):
  31 + for ts in tile_scheme:
  32 + if ts.get("isDefault"):
  33 + default_scheme = ts
  34 + else:
  35 + other_scheme.append(ts)
  36 + else:
  37 + default_scheme = tile_scheme
  38 +
  39 + project_xml_format = '''
  40 +<?xml version="1.0"?>
  41 +<dmap projectname="wmtstest" version="4.0">
  42 + <projectCrs>
  43 + <spatialrefsys>
  44 + <wkt>{wkt}</wkt>
  45 + <proj4>{proj4}</proj4>
  46 + <srid>{srid}</srid>
  47 + <description/>
  48 + <projectionacronym/>
  49 + </spatialrefsys>
  50 + </projectCrs>
  51 + <projectlayers>
  52 + <maplayer name="{name}" alias="{alias}" type="0">
  53 + <extent>
  54 + <xmin>{xmin}</xmin>
  55 + <ymin>{ymin}</ymin>
  56 + <xmax>{xmax}</xmax>
  57 + <ymax>{ymax}</ymax>
  58 + </extent>
  59 + <style>{layer_style}</style>
  60 + <format>{layer_format}</format>
  61 + <vendor>{vendor}</vendor>
  62 + <datasource>{datasource}</datasource>
  63 + <defaultTileMatrixSet>{defaultTileMatrixSet}</defaultTileMatrixSet>
  64 + <tileMatrixSets>
  65 + <tileMatrixSet>
  66 + <id>{id}</id>
  67 + <crs>{crs}</crs>
  68 + <tileCols>{cols}</tileCols>
  69 + <tileRows>{rows}</tileRows>
  70 + <dpi>{dpi}</dpi>
  71 + <tileOrigin>
  72 + <X>{x}</X>
  73 + <Y>{y}</Y>
  74 + </tileOrigin>
  75 + <levels>
  76 + {levels}
  77 + </levels>
  78 + </tileMatrixSet>
  79 + {other_tileMatrixSet}
  80 + </tileMatrixSets>
  81 + </maplayer>
  82 + </projectlayers>
  83 +</dmap>
  84 +'''
  85 +
  86 + level_each = '''<level>
  87 + <id>{lev}</id>
  88 + <scaleDenominator>{scale}</scaleDenominator>
  89 + <resolution>{resolution}</resolution>
  90 + </level>
  91 + '''
  92 +
  93 + levels = ''
  94 + for level in json.loads(default_scheme.get("levels")):
  95 + levels = "{}{}".format(levels, level_each.format(lev=level["level"],
  96 + scale=level["scale"],
  97 + resolution=level["resolution"],
  98 + ))
  99 + #删除空格
  100 + levels = levels.strip()
  101 +
  102 + layer_extent = para.get("layer_extent").split(",")
  103 +
  104 + project_xml = project_xml_format.format(wkt="",
  105 + proj4="",
  106 + srid=para.get("crs").split("::")[-1],
  107 + name=para.get("layer_name"),
  108 + alias=para.get("alias") if para.get("alias") else "",
  109 + xmin=layer_extent[0],
  110 + xmax=layer_extent[1],
  111 + ymin=layer_extent[2],
  112 + ymax=layer_extent[3],
  113 + layer_style=para.get("layer_style"),
  114 + layer_format=para.get("layer_format"),
  115 + vendor=para.get("vendor"),
  116 + datasource=para.get("datasource"),
  117 + id=default_scheme.get("name"),
  118 + crs=para.get("crs"),
  119 + cols=default_scheme.get("cols"),
  120 + rows=default_scheme.get("rows"),
  121 + dpi=default_scheme.get("dpi"),
  122 + x=default_scheme.get("top_left").split(",")[0],
  123 + y=default_scheme.get("top_left").split(",")[1],
  124 + levels=levels,
  125 + other_tileMatrixSet=cls.create_other(other_scheme),
  126 + defaultTileMatrixSet=default_scheme.get("name"),
  127 + )
  128 + else:
  129 +
  130 + project_xml_format = '''
  131 +<?xml version="1.0"?>
  132 +<dmap projectname="tmstest" version="4.0">
  133 + <projectCrs>
  134 + <spatialrefsys>
  135 + <wkt>{wkt}</wkt>
  136 + <proj4>{proj4}</proj4>
  137 + <srid>{srid}</srid>
  138 + <description/>
  139 + <projectionacronym/>
  140 + </spatialrefsys>
  141 + </projectCrs>
  142 + <projectlayers>
  143 + <maplayer name="{name}" alias="{alias}" type="0">
  144 + <extent>
  145 + <xmin>{xmin}</xmin>
  146 + <ymin>{ymin}</ymin>
  147 + <xmax>{xmax}</xmax>
  148 + <ymax>{ymax}</ymax>
  149 + </extent>
  150 + <style>{layer_style}</style>
  151 + <format>{layer_format}</format>
  152 + <vendor>{vendor}</vendor>
  153 + <datasource>{datasource}</datasource>
  154 + </maplayer>
  155 + </projectlayers>
  156 +</dmap>
  157 +'''
  158 + layer_extent = para.get("layer_extent").split(",")
  159 + project_xml = project_xml_format.format(wkt="",
  160 + proj4="",
  161 + srid=para.get("crs").split("::")[-1],
  162 + name=para.get("layer_name"),
  163 + alias=para.get("alias") if para.get("alias") else "",
  164 + xmin=layer_extent[0],
  165 + xmax=layer_extent[1],
  166 + ymin=layer_extent[2],
  167 + ymax=layer_extent[3],
  168 + layer_style=para.get("layer_style"),
  169 + layer_format=para.get("layer_format"),
  170 + vendor=para.get("vendor"),
  171 + datasource=para.get("datasource"),
  172 + )
  173 +
  174 + project_xml = project_xml.strip()
  175 + StructurePrint().print(project_xml)
  176 + StructurePrint().print(str(base64.b64encode(project_xml.encode('utf-8')), encoding="utf8"))
  177 + return str(base64.b64encode(project_xml.encode('utf-8')), encoding="utf8")
  178 +
  179 + @classmethod
  180 + def create_other(cls,other_scheme):
  181 + other = ""
  182 +
  183 + for scheme in other_scheme:
  184 +
  185 + tileMatrixSet='''
  186 + <tileMatrixSet>
  187 + <id>{id}</id>
  188 + <crs>{crs}</crs>
  189 + <tileCols>{cols}</tileCols>
  190 + <tileRows>{rows}</tileRows>
  191 + <dpi>{dpi}</dpi>
  192 + <tileOrigin>
  193 + <X>{x}</X>
  194 + <Y>{y}</Y>
  195 + </tileOrigin>
  196 + <levels>
  197 + {levels}
  198 + </levels>
  199 + </tileMatrixSet>
  200 + '''
  201 +
  202 + level_each = '''<level>
  203 + <id>{lev}</id>
  204 + <scaleDenominator>{scale}</scaleDenominator>
  205 + <resolution>{resolution}</resolution>
  206 + </level>
  207 + '''
  208 +
  209 + levels = ''
  210 + for level in json.loads(scheme.get("levels")):
  211 + levels = "{}{}".format(levels, level_each.format(lev=level["level"],
  212 + scale=level["scale"],
  213 + resolution=level["resolution"],
  214 + ))
  215 + # 删除空格
  216 + levels = levels.strip()
  217 + tileMatrixSet = tileMatrixSet.format(id=scheme.get("name"),
  218 + crs=scheme.get("crs"),
  219 + cols=scheme.get("tileCols"),
  220 + rows=scheme.get("tileRows"),
  221 + dpi=scheme.get("dpi"),
  222 + x=scheme.get("top_left").split(",")[0],
  223 + y=scheme.get("top_left").split(",")[1],
  224 + levels=levels
  225 + )
  226 + tileMatrixSet = tileMatrixSet.strip()
  227 + other+=tileMatrixSet
  228 +
  229 + other = other.strip()
  230 + return other
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2022/4/12
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +from osgeo.ogr import *
  8 +from osgeo import ogr
  9 +import time
  10 +import shutil
  11 +import sys
  12 +import os
  13 +
  14 +def shp2mvt(data_path,out_path):
  15 + t1 = time.time()
  16 +
  17 + if os.path.exists(out_path):
  18 + shutil.rmtree(out_path, True)
  19 + try:
  20 + data_path = sys.argv[1]
  21 + out_path = sys.argv[2]
  22 + except:
  23 + pass
  24 +
  25 + driver: Driver = ogr.GetDriverByName("ESRI Shapefile")
  26 + ds: DataSource = driver.Open(data_path, 0)
  27 +
  28 + lay:Layer = ds.GetLayer(0)
  29 +
  30 +
  31 + mvt_driver:Driver = ogr.GetDriverByName("MVT")
  32 +
  33 + mvt_driver.CopyDataSource(ds,out_path)
  34 +
  35 + print(time.time()-t1)
  36 +
  37 +
  38 +def shp2geojson(data_path,out_path,format="pbf"):
  39 + t1 = time.time()
  40 +
  41 + try:
  42 + data_path = sys.argv[1]
  43 + out_path = sys.argv[2]
  44 + except:
  45 + pass
  46 +
  47 +
  48 + driver: Driver = ogr.GetDriverByName("ESRI Shapefile")
  49 + ds: DataSource = driver.Open(data_path, 0)
  50 +
  51 + mvt_driver:Driver = ogr.GetDriverByName("GeoJSON")
  52 +
  53 + mvt_driver.CopyDataSource(ds,out_path)
  54 +
  55 + print(time.time()-t1)
  56 +
  57 +def geojson2mvt(data_path,out_path):
  58 + t1 = time.time()
  59 +
  60 + if os.path.exists(out_path):
  61 + shutil.rmtree(out_path, True)
  62 +
  63 +
  64 + geosjon_driver: Driver = ogr.GetDriverByName("GeoJSON")
  65 + ds: DataSource = geosjon_driver.Open(data_path, 0)
  66 +
  67 + mvt_driver:Driver = ogr.GetDriverByName("MVT")
  68 +
  69 + mvt_driver.CopyDataSource(ds,out_path)
  70 +
  71 + print(time.time()-t1)
  72 +
  73 +if __name__ == '__main__':
  74 +
  75 +
  76 + # shp2geojson(r"J:\Data\矢量数据\广州\gzpoi\gzpoisub3.shp",r"J:\Data\矢量数据\广州\gzpoi\gzpoisub3.geojson")
  77 +
  78 + # geojson2mvt(r"J:\Data\矢量数据\广州\gzpoi\gzpoisub2.geojson",r"J:\Data\矢量数据\广州\gzpoi\gzpoisub3.mvt")
  79 + shp2mvt(r"J:\Data\矢量数据\广州\gzpoi\gzpoisub3.shp",r"J:\Data\矢量数据\广州\gzpoi\gzpoisub3.mvt")
\ No newline at end of file
... ...
请 注册 或 登录 后发表评论