HWJ 何伟俊 / DMapManager · 提交

提交 1a2a27960d57cbc920265b7e7a9b45190945b047

作者 nheweijun
1 个父辈 170b08aa

2021.12.01 完成金字塔接口，准备优化入库队列

内嵌 并排对比
正在显示 45 个修改的文件 包含 506 行增加 和 444 行删除
@@ -27,11 +27,6 @@ class Api(ApiTemplate): @@ -27,11 +27,6 @@ class Api(ApiTemplate):
27 if not Database.query.filter_by(guid=self.para.get("database_guid")).one_or_none(): 27 if not Database.query.filter_by(guid=self.para.get("database_guid")).one_or_none():
28 res["msg"]="数据库不存在！" 28 res["msg"]="数据库不存在！"
29 return res 29 return res
30 -  
31 -  
32 - # 可以创建已有数据目录的子目录  
33 - # if Table.query.filter_by(catalog_guid=self.para.get("pguid")).all():  
34 - # raise Exception("父目录挂载了数据，不能创建子目录")  
35 30
36 if Catalog.query.filter_by(name=self.para.get("name"), 31 if Catalog.query.filter_by(name=self.para.get("name"),
37 pguid=self.para.get("pguid"), 32 pguid=self.para.get("pguid"),
@@ -39,10 +34,6 @@ class Api(ApiTemplate): @@ -39,10 +34,6 @@ class Api(ApiTemplate):
39 res["msg"]="目录已经存在！" 34 res["msg"]="目录已经存在！"
40 return res 35 return res
41 36
42 -  
43 - # if Catalog.query.filter_by(pguid="0",database_guid=self.para.get("database_guid")).one_or_none() and self.para.get("pguid").__eq__("0"):  
44 - # raise Exception("只能有一个根目录！")  
45 -  
46 guid = uuid.uuid1().__str__() 37 guid = uuid.uuid1().__str__()
47 path = guid 38 path = guid
48 39
@@ -16,14 +16,6 @@ class Api(ApiTemplate): @@ -16,14 +16,6 @@ class Api(ApiTemplate):
16 res = {} 16 res = {}
17 try: 17 try:
18 # 业务逻辑 18 # 业务逻辑
19 -  
20 -  
21 - # 啥情况都能删  
22 - # if Table.query.filter_by(catalog_guid=self.para.get("guid")).all():  
23 - # raise Exception("目录挂载了数据，不可删除，可将数据移出目录后删除！")  
24 - # if Catalog.query.filter_by(pguid=self.para.get("guid")).all():  
25 - # raise Exception("目录非子目录，不可删除，请先将子目录删除!")  
26 -  
27 catalog_guid = self.para.get("guid") 19 catalog_guid = self.para.get("guid")
28 20
29 catalog = Catalog.query.filter_by(guid=catalog_guid).one_or_none() 21 catalog = Catalog.query.filter_by(guid=catalog_guid).one_or_none()
@@ -32,20 +24,7 @@ class Api(ApiTemplate): @@ -32,20 +24,7 @@ class Api(ApiTemplate):
32 return res 24 return res
33 25
34 else: 26 else:
35 -  
36 # 转移目录下的数据 27 # 转移目录下的数据
37 -  
38 - # # 删除根节点  
39 - # if catalog.pguid.__eq__("0"):  
40 - # database_guid = catalog.database_guid  
41 - # Table.query.filter_by(database_guid=database_guid).update({"catalog_guid": None})  
42 - # catalogs = Catalog.query.filter(Catalog.path.like("%" + catalog_guid + "%")).all()  
43 - # for cata in catalogs:  
44 - # db.session.delete(cata)  
45 - #  
46 - # # 获取所有子目录：  
47 - # else:  
48 -  
49 pguid = catalog.pguid 28 pguid = catalog.pguid
50 # 所有目录 29 # 所有目录
51 catalogs = Catalog.query.filter(Catalog.path.like("%" + catalog_guid + "%")).all() 30 catalogs = Catalog.query.filter(Catalog.path.like("%" + catalog_guid + "%")).all()
@@ -9,7 +9,6 @@ from app.util.component.ApiTemplate import ApiTemplate @@ -9,7 +9,6 @@ from app.util.component.ApiTemplate import ApiTemplate
9 class Api(ApiTemplate): 9 class Api(ApiTemplate):
10 api_name = "修改目录" 10 api_name = "修改目录"
11 def process(self): 11 def process(self):
12 -  
13 12
14 # 返回结果 13 # 返回结果
15 res = {} 14 res = {}
@@ -22,8 +22,7 @@ class Api(ApiTemplate): @@ -22,8 +22,7 @@ class Api(ApiTemplate):
22 for cata in catalogs: 22 for cata in catalogs:
23 catalog_guids = [c.guid for c in Catalog.query.filter(Catalog.path.like("%" + cata.guid + "%")).all()] 23 catalog_guids = [c.guid for c in Catalog.query.filter(Catalog.path.like("%" + cata.guid + "%")).all()]
24 table_count = Table.query.filter(Table.catalog_guid.in_(catalog_guids)).count() 24 table_count = Table.query.filter(Table.catalog_guid.in_(catalog_guids)).count()
25 -  
26 - # cata_json = object_to_json(cata) 25 +
27 cata_json ={} 26 cata_json ={}
28 cata_json["database_guid"]=cata.database_guid 27 cata_json["database_guid"]=cata.database_guid
29 cata_json["description"] = cata.description 28 cata_json["description"] = cata.description
@@ -21,8 +21,7 @@ class Api(ApiTemplate): @@ -21,8 +21,7 @@ class Api(ApiTemplate):
21 for cata in catalogs: 21 for cata in catalogs:
22 catalog_guids = [c.guid for c in Catalog.query.filter(Catalog.path.like("%" + cata.guid + "%")).all()] 22 catalog_guids = [c.guid for c in Catalog.query.filter(Catalog.path.like("%" + cata.guid + "%")).all()]
23 table_count = Table.query.filter(Table.catalog_guid.in_(catalog_guids)).count() 23 table_count = Table.query.filter(Table.catalog_guid.in_(catalog_guids)).count()
24 -  
25 - # cata_json = object_to_json(cata) 24 +
26 cata_json ={} 25 cata_json ={}
27 cata_json["database_guid"]=cata.database_guid 26 cata_json["database_guid"]=cata.database_guid
28 cata_json["description"] = cata.description 27 cata_json["description"] = cata.description
@@ -245,11 +245,6 @@ class Api(ApiTemplate): @@ -245,11 +245,6 @@ class Api(ApiTemplate):
245 245
246 if l_name.__contains__("_vacuate_"): 246 if l_name.__contains__("_vacuate_"):
247 247
248 - # 没有权限的表跳过  
249 - # if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds):  
250 - # StructurePrint().print("用户{}对表{}没有select权限！".format(db_tuple[0], l_name), "warn")  
251 - # continue  
252 -  
253 base_layer_name=l_name.split("_vacuate_")[1] 248 base_layer_name=l_name.split("_vacuate_")[1]
254 level = l_name.split("_")[-2] 249 level = l_name.split("_")[-2]
255 pixel_distance_str: str ="0" 250 pixel_distance_str: str ="0"
@@ -28,18 +28,12 @@ class Api(ApiTemplate): @@ -28,18 +28,12 @@ class Api(ApiTemplate):
28 encryption = int(self.para.get("encryption", "0")) 28 encryption = int(self.para.get("encryption", "0"))
29 if encryption: 29 if encryption:
30 passwd = DES.decode(passwd) 30 passwd = DES.decode(passwd)
31 -  
32 - 31 +
33 sqlalchemy_uri = "postgresql://{}:{}@{}:{}/{}".format(user,passwd,host,port,database) 32 sqlalchemy_uri = "postgresql://{}:{}@{}:{}/{}".format(user,passwd,host,port,database)
34 -  
35 - # if is_encrypt == "1":  
36 - # sqlalchemy_uri=DES.decode(sqlalchemy_uri)  
37 - 33 +
38 engine = create_engine(sqlalchemy_uri, connect_args={'connect_timeout': 2}) 34 engine = create_engine(sqlalchemy_uri, connect_args={'connect_timeout': 2})
39 with closing(engine.connect()): 35 with closing(engine.connect()):
40 pass 36 pass
41 -  
42 -  
43 #判断数据库是否存在 37 #判断数据库是否存在
44 datab = db.session.query(Database).filter_by(alias=self.para.get("alias")).one_or_none() 38 datab = db.session.query(Database).filter_by(alias=self.para.get("alias")).one_or_none()
45 #真实的数据库 39 #真实的数据库
@@ -85,9 +79,6 @@ class Api(ApiTemplate): @@ -85,9 +79,6 @@ class Api(ApiTemplate):
85 api_doc={ 79 api_doc={
86 "tags":["数据库接口"], 80 "tags":["数据库接口"],
87 "parameters":[ 81 "parameters":[
88 - # {"name": "sqlalchemy_uri",  
89 - # "in": "formData",  
90 - # "type": "string","description":"数据库uri","required": "true"},  
91 {"name": "host", 82 {"name": "host",
92 "in": "formData", 83 "in": "formData",
93 "type": "string", "required": "true"}, 84 "type": "string", "required": "true"},
@@ -103,9 +94,6 @@ class Api(ApiTemplate): @@ -103,9 +94,6 @@ class Api(ApiTemplate):
103 {"name": "database", 94 {"name": "database",
104 "in": "formData", 95 "in": "formData",
105 "type": "string", "required": "true"}, 96 "type": "string", "required": "true"},
106 - # {"name": "alias",  
107 - # "in": "formData",  
108 - # "type": "string", "required": "true"},  
109 {"name": "encryption", 97 {"name": "encryption",
110 "in": "formData", 98 "in": "formData",
111 "type": "int", "description": "密码是否加密", "enum": [0, 1]}, 99 "type": "int", "description": "密码是否加密", "enum": [0, 1]},
@@ -76,8 +76,6 @@ class Api(ApiTemplate): @@ -76,8 +76,6 @@ class Api(ApiTemplate):
76 dirpath = os.path.join(parent, "file_tmp", uuid_) 76 dirpath = os.path.join(parent, "file_tmp", uuid_)
77 os.makedirs(dirpath) 77 os.makedirs(dirpath)
78 data_source: DataSource = driver.CreateDataSource(dirpath + "/{}.shp".format(table_name)) 78 data_source: DataSource = driver.CreateDataSource(dirpath + "/{}.shp".format(table_name))
79 - # data_source.CopyLayer(layer, table_name)  
80 -  
81 79
82 fid = layer.GetFIDColumn() 80 fid = layer.GetFIDColumn()
83 pg_layer: Layer = data_source.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType()) 81 pg_layer: Layer = data_source.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType())
@@ -122,19 +120,13 @@ class Api(ApiTemplate): @@ -122,19 +120,13 @@ class Api(ApiTemplate):
122 field_defn.SetAlternativeName(field_alias) 120 field_defn.SetAlternativeName(field_alias)
123 121
124 table_alias= table.alias 122 table_alias= table.alias
125 -  
126 - # if is_chinese(table_name):  
127 - # if not table_alias:  
128 - # table_alias = table_name  
129 - # table_name = "table{}".format(table_name.__hash__()) 123 +
130 124
131 fid = layer.GetFIDColumn() 125 fid = layer.GetFIDColumn()
132 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(),["LAYER_ALIAS={}".format(table_alias)]) 126 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(),["LAYER_ALIAS={}".format(table_alias)])
133 schema = [sche for sche in layer.schema if not sche.name.__eq__(fid)] 127 schema = [sche for sche in layer.schema if not sche.name.__eq__(fid)]
134 - # schema = layer.schema  
135 pg_layer.CreateFields(schema) 128 pg_layer.CreateFields(schema)
136 129
137 -  
138 # gdb 不支持fid=0的要素，所以识别到后要+1 130 # gdb 不支持fid=0的要素，所以识别到后要+1
139 offset = 0 131 offset = 0
140 f1:Feature = layer.GetNextFeature() 132 f1:Feature = layer.GetNextFeature()
@@ -146,9 +138,6 @@ class Api(ApiTemplate): @@ -146,9 +138,6 @@ class Api(ApiTemplate):
146 feature.SetFID(feature.GetFID()+offset) 138 feature.SetFID(feature.GetFID()+offset)
147 pg_layer.CreateFeature(feature) 139 pg_layer.CreateFeature(feature)
148 140
149 -  
150 - # gdb_ds.CopyLayer(layer, table_name,["LAYER_ALIAS={}".format(table_alias)])  
151 -  
152 gdb_ds.Destroy() 141 gdb_ds.Destroy()
153 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path]) 142 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path])
154 data.append({"name": ",".join(table_names), "download_url": "http://" + configure.deploy_ip_host + "/API/IO/Download/{}".format(uuid_+".gdb" + ".zip")}) 143 data.append({"name": ",".join(table_names), "download_url": "http://" + configure.deploy_ip_host + "/API/IO/Download/{}".format(uuid_+".gdb" + ".zip")})
@@ -16,6 +16,7 @@ import configure @@ -16,6 +16,7 @@ import configure
16 from app.util.component.ApiTemplate import ApiTemplate 16 from app.util.component.ApiTemplate import ApiTemplate
17 from app.util.component.PGUtil import PGUtil 17 from app.util.component.PGUtil import PGUtil
18 from app.util.component.ZipUtil import ZipUtil 18 from app.util.component.ZipUtil import ZipUtil
  19 +from app.util.component.StructuredPrint import StructurePrint
19 import multiprocessing 20 import multiprocessing
20 import datetime 21 import datetime
21 22
@@ -29,6 +30,7 @@ class Api(ApiTemplate): @@ -29,6 +30,7 @@ class Api(ApiTemplate):
29 30
30 task_guid = uuid.uuid1().__str__() 31 task_guid = uuid.uuid1().__str__()
31 download_process = multiprocessing.Process(target=self.download, args=(task_guid,self.para)) 32 download_process = multiprocessing.Process(target=self.download, args=(task_guid,self.para))
  33 + download_process.start()
32 34
33 task = Task(guid=task_guid, 35 task = Task(guid=task_guid,
34 name="{}下载".format(self.para.get("table_name")), 36 name="{}下载".format(self.para.get("table_name")),
@@ -38,14 +40,15 @@ class Api(ApiTemplate): @@ -38,14 +40,15 @@ class Api(ApiTemplate):
38 creator=self.para.get("creator"), 40 creator=self.para.get("creator"),
39 file_name=None, 41 file_name=None,
40 process="数据下载中", 42 process="数据下载中",
41 - database_guid=self.para.get("database_guid")) 43 + database_guid=self.para.get("database_guid"),
  44 + task_pid=download_process.pid)
42 45
43 db.session.add(task) 46 db.session.add(task)
44 db.session.commit() 47 db.session.commit()
45 - download_process.start() 48 +
46 49
47 res["data"] = "下载任务已提交!" 50 res["data"] = "下载任务已提交!"
48 - res["state"] = True 51 + res["result"] = True
49 52
50 except Exception as e: 53 except Exception as e:
51 raise e 54 raise e
@@ -130,7 +133,7 @@ class Api(ApiTemplate): @@ -130,7 +133,7 @@ class Api(ApiTemplate):
130 dirpath = os.path.join(parent, "file_tmp", uuid_) 133 dirpath = os.path.join(parent, "file_tmp", uuid_)
131 os.makedirs(dirpath) 134 os.makedirs(dirpath)
132 data_source: DataSource = driver.CreateDataSource(dirpath + "/{}.shp".format(table_name)) 135 data_source: DataSource = driver.CreateDataSource(dirpath + "/{}.shp".format(table_name))
133 - # data_source.CopyLayer(layer, table_name) 136 +
134 137
135 fid = layer.GetFIDColumn() 138 fid = layer.GetFIDColumn()
136 pg_layer: Layer = data_source.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType()) 139 pg_layer: Layer = data_source.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType())
@@ -138,8 +141,13 @@ class Api(ApiTemplate): @@ -138,8 +141,13 @@ class Api(ApiTemplate):
138 141
139 pg_layer.CreateFields(schema) 142 pg_layer.CreateFields(schema)
140 layer.ResetReading() 143 layer.ResetReading()
  144 +
  145 + count = 0
141 for feature in layer: 146 for feature in layer:
142 pg_layer.CreateFeature(feature) 147 pg_layer.CreateFeature(feature)
  148 + count += 1
  149 + if count % 10000 == 0:
  150 + StructurePrint().print("{}图层已下载{}个对象".format(table_name, count))
143 151
144 data_source.Destroy() 152 data_source.Destroy()
145 153
@@ -150,6 +158,7 @@ class Api(ApiTemplate): @@ -150,6 +158,7 @@ class Api(ApiTemplate):
150 158
151 159
152 def download_gdb(self,sys_session,table_names,ds,database_guid): 160 def download_gdb(self,sys_session,table_names,ds,database_guid):
  161 +
153 ogr.RegisterAll() 162 ogr.RegisterAll()
154 data = [] 163 data = []
155 gdal.UseExceptions() 164 gdal.UseExceptions()
@@ -176,11 +185,6 @@ class Api(ApiTemplate): @@ -176,11 +185,6 @@ class Api(ApiTemplate):
176 field_defn.SetAlternativeName(field_alias) 185 field_defn.SetAlternativeName(field_alias)
177 186
178 table_alias= table.alias 187 table_alias= table.alias
179 -  
180 - # if is_chinese(table_name):  
181 - # if not table_alias:  
182 - # table_alias = table_name  
183 - # table_name = "table{}".format(table_name.__hash__())  
184 188
185 fid = layer.GetFIDColumn() 189 fid = layer.GetFIDColumn()
186 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(),["LAYER_ALIAS={}".format(table_alias)]) 190 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(),["LAYER_ALIAS={}".format(table_alias)])
@@ -196,12 +200,13 @@ class Api(ApiTemplate): @@ -196,12 +200,13 @@ class Api(ApiTemplate):
196 if f1.GetFID().__eq__(0): 200 if f1.GetFID().__eq__(0):
197 offset = 1 201 offset = 1
198 layer.ResetReading() 202 layer.ResetReading()
  203 + count = 0
199 for feature in layer: 204 for feature in layer:
200 feature.SetFID(feature.GetFID()+offset) 205 feature.SetFID(feature.GetFID()+offset)
201 pg_layer.CreateFeature(feature) 206 pg_layer.CreateFeature(feature)
202 -  
203 -  
204 - # gdb_ds.CopyLayer(layer, table_name,["LAYER_ALIAS={}".format(table_alias)]) 207 + count += 1
  208 + if count % 10000 == 0:
  209 + StructurePrint().print("{}图层已下载{}个对象".format(table_name, count))
205 210
206 gdb_ds.Destroy() 211 gdb_ds.Destroy()
207 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path]) 212 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path])
@@ -29,9 +29,6 @@ def data_entry_center(): @@ -29,9 +29,6 @@ def data_entry_center():
29 29
30 # 已经结束的进程 从监测中删除 30 # 已经结束的进程 从监测中删除
31 remove_process = [] 31 remove_process = []
32 -  
33 - # structured_print(running_dict.__len__().__str__())  
34 -  
35 for process, layer_names in running_dict.items(): 32 for process, layer_names in running_dict.items():
36 if not process.is_alive(): 33 if not process.is_alive():
37 for l in layer_names: 34 for l in layer_names:
@@ -44,15 +41,13 @@ def data_entry_center(): @@ -44,15 +41,13 @@ def data_entry_center():
44 for process in remove_process: 41 for process in remove_process:
45 running_dict.pop(process) 42 running_dict.pop(process)
46 43
47 - # StructurePrint().print("listening...")  
48 44
49 # 入库进程少于阈值，开启入库进程 45 # 入库进程少于阈值，开启入库进程
50 -  
51 inter_size = sys_session.query( 46 inter_size = sys_session.query(
52 distinct(InsertingLayerName.task_guid)).count() 47 distinct(InsertingLayerName.task_guid)).count()
53 48
54 if inter_size < configure.entry_data_thread: 49 if inter_size < configure.entry_data_thread:
55 - # 锁表啊 50 + # 锁表
56 ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( 51 ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by(
57 Task.create_time).with_lockmode("update").limit(1).one_or_none() 52 Task.create_time).with_lockmode("update").limit(1).one_or_none()
58 if ready_task: 53 if ready_task:
@@ -100,7 +95,13 @@ def data_entry_center(): @@ -100,7 +95,13 @@ def data_entry_center():
100 entry_data_process = multiprocessing.Process( 95 entry_data_process = multiprocessing.Process(
101 target=EntryDataVacuate().entry, args=(parameter,)) 96 target=EntryDataVacuate().entry, args=(parameter,))
102 entry_data_process.start() 97 entry_data_process.start()
  98 +
  99 + pid = entry_data_process.pid
  100 + sys_session.query(Task).filter_by(guid=ready_task.guid).update({"task_pid":pid})
  101 + sys_session.commit()
  102 +
103 running_dict[entry_data_process] = this_task_layer 103 running_dict[entry_data_process] = this_task_layer
  104 +
104 except Exception as e: 105 except Exception as e:
105 sys_session.query(Task).filter_by(guid=ready_task.guid).update( 106 sys_session.query(Task).filter_by(guid=ready_task.guid).update(
106 {"state": -1, "process": "入库失败"}) 107 {"state": -1, "process": "入库失败"})
1 -GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]  
1 -<?xml version="1.0" encoding="UTF-8"?>  
2 -<metadata xml:lang="zh"><Esri><CreaDate>20170418</CreaDate><CreaTime>19355600</CreaTime><ArcGISFormat>1.0</ArcGISFormat><SyncOnce>FALSE</SyncOnce><DataProperties><itemProps><itemName Sync="TRUE">北京县区</itemName><imsContentType Sync="TRUE">002</imsContentType><itemLocation><linkage Sync="TRUE">file://\\4N\E$\Data\北京县区区划\北京县区.shp</linkage><protocol Sync="TRUE">Local Area Network</protocol></itemLocation><nativeExtBox><westBL Sync="TRUE">115.417282</westBL><eastBL Sync="TRUE">117.500126</eastBL><southBL Sync="TRUE">39.438282</southBL><northBL Sync="TRUE">41.059244</northBL><exTypeCode Sync="TRUE">1</exTypeCode></nativeExtBox><itemSize Sync="TRUE">0.270</itemSize></itemProps><coordRef><type Sync="TRUE">Geographic</type><geogcsn Sync="TRUE">GCS_WGS_1984</geogcsn><csUnits Sync="TRUE">Angular Unit: Degree (0.017453)</csUnits><peXml Sync="TRUE">&lt;GeographicCoordinateSystem xsi:type='typens:GeographicCoordinateSystem' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.1'&gt;&lt;WKT&gt;GEOGCS[&amp;quot;GCS_WGS_1984&amp;quot;,DATUM[&amp;quot;D_WGS_1984&amp;quot;,SPHEROID[&amp;quot;WGS_1984&amp;quot;,6378137.0,298.257223563]],PRIMEM[&amp;quot;Greenwich&amp;quot;,0.0],UNIT[&amp;quot;Degree&amp;quot;,0.0174532925199433],AUTHORITY[&amp;quot;EPSG&amp;quot;,4326]]&lt;/WKT&gt;&lt;XOrigin&gt;-400&lt;/XOrigin&gt;&lt;YOrigin&gt;-400&lt;/YOrigin&gt;&lt;XYScale&gt;11258999068426.238&lt;/XYScale&gt;&lt;ZOrigin&gt;-100000&lt;/ZOrigin&gt;&lt;ZScale&gt;10000&lt;/ZScale&gt;&lt;MOrigin&gt;-100000&lt;/MOrigin&gt;&lt;MScale&gt;10000&lt;/MScale&gt;&lt;XYTolerance&gt;8.983152841195215e-009&lt;/XYTolerance&gt;&lt;ZTolerance&gt;0.001&lt;/ZTolerance&gt;&lt;MTolerance&gt;0.001&lt;/MTolerance&gt;&lt;HighPrecision&gt;true&lt;/HighPrecision&gt;&lt;LeftLongitude&gt;-180&lt;/LeftLongitude&gt;&lt;WKID&gt;4326&lt;/WKID&gt;&lt;LatestWKID&gt;4326&lt;/LatestWKID&gt;&lt;/GeographicCoordinateSystem&gt;</peXml></coordRef></DataProperties><SyncDate>20210316</SyncDate><SyncTime>11462400</SyncTime><ModDate>20210316</ModDate><ModTime>11462400</ModTime></Esri><dataIdInfo><envirDesc Sync="TRUE"> Version 6.2 (Build 9200) ; Esri ArcGIS 10.2.0.3348</envirDesc><dataLang><languageCode value="zho" Sync="TRUE"></languageCode><countryCode value="CHN" Sync="TRUE"></countryCode></dataLang><idCitation><resTitle Sync="TRUE">北京县区</resTitle><presForm><PresFormCd value="005" Sync="TRUE"></PresFormCd></presForm></idCitation><spatRpType><SpatRepTypCd value="001" Sync="TRUE"></SpatRepTypCd></spatRpType><dataExt><geoEle><GeoBndBox esriExtentType="search"><exTypeCode Sync="TRUE">1</exTypeCode><westBL Sync="TRUE">115.417282</westBL><eastBL Sync="TRUE">117.500126</eastBL><northBL Sync="TRUE">41.059244</northBL><southBL Sync="TRUE">39.438282</southBL></GeoBndBox></geoEle></dataExt></dataIdInfo><mdLang><languageCode value="zho" Sync="TRUE"></languageCode><countryCode value="CHN" Sync="TRUE"></countryCode></mdLang><distInfo><distFormat><formatName Sync="TRUE">Shapefile</formatName></distFormat><distTranOps><transSize Sync="TRUE">0.270</transSize></distTranOps></distInfo><mdHrLv><ScopeCd value="005" Sync="TRUE"></ScopeCd></mdHrLv><mdHrLvName Sync="TRUE">dataset</mdHrLvName><refSysInfo><RefSystem><refSysID><identCode code="4326" Sync="TRUE"></identCode><idCodeSpace Sync="TRUE">EPSG</idCodeSpace><idVersion Sync="TRUE">8.1.1</idVersion></refSysID></RefSystem></refSysInfo><spatRepInfo><VectSpatRep><geometObjs Name="北京县区"><geoObjTyp><GeoObjTypCd value="002" Sync="TRUE"></GeoObjTypCd></geoObjTyp><geoObjCnt Sync="TRUE">17</geoObjCnt></geometObjs><topLvl><TopoLevCd value="001" Sync="TRUE"></TopoLevCd></topLvl></VectSpatRep></spatRepInfo><spdoinfo><ptvctinf><esriterm Name="北京县区"><efeatyp Sync="TRUE">Simple</efeatyp><efeageom code="4" Sync="TRUE"></efeageom><esritopo Sync="TRUE">FALSE</esritopo><efeacnt Sync="TRUE">17</efeacnt><spindex Sync="TRUE">TRUE</spindex><linrefer Sync="TRUE">FALSE</linrefer></esriterm></ptvctinf></spdoinfo><eainfo><detailed Name="北京县区"><enttyp><enttypl Sync="TRUE">北京县区</enttypl><enttypt Sync="TRUE">Feature Class</enttypt><enttypc Sync="TRUE">17</enttypc></enttyp><attr><attrlabl Sync="TRUE">FID</attrlabl><attalias Sync="TRUE">FID</attalias><attrtype Sync="TRUE">OID</attrtype><attwidth Sync="TRUE">4</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Internal feature number.</attrdef><attrdefs Sync="TRUE">Esri</attrdefs><attrdomv><udom Sync="TRUE">Sequential unique whole numbers that are automatically generated.</udom></attrdomv></attr><attr><attrlabl Sync="TRUE">OBJECTID</attrlabl><attalias Sync="TRUE">OBJECTID</attalias><attrtype Sync="TRUE">Double</attrtype><attwidth Sync="TRUE">10</attwidth><atprecis Sync="TRUE">10</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Internal feature number.</attrdef><attrdefs Sync="TRUE">Esri</attrdefs><attrdomv><udom Sync="TRUE">Sequential unique whole numbers that are automatically generated.</udom></attrdomv></attr><attr><attrlabl Sync="TRUE">Shape</attrlabl><attalias Sync="TRUE">Shape</attalias><attrtype Sync="TRUE">Geometry</attrtype><attwidth Sync="TRUE">0</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Feature geometry.</attrdef><attrdefs Sync="TRUE">Esri</attrdefs><attrdomv><udom Sync="TRUE">Coordinates defining the features.</udom></attrdomv></attr><attr><attrlabl Sync="TRUE">NAME</attrlabl><attalias Sync="TRUE">NAME</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">60</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">KIND</attrlabl><attalias Sync="TRUE">KIND</attalias><attrtype Sync="TRUE">String</attrtype><attwidth Sync="TRUE">4</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">Shape_Leng</attrlabl><attalias Sync="TRUE">Shape_Leng</attalias><attrtype Sync="TRUE">Double</attrtype><attwidth Sync="TRUE">19</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale></attr><attr><attrlabl Sync="TRUE">Shape_Area</attrlabl><attalias Sync="TRUE">Shape_Area</attalias><attrtype Sync="TRUE">Double</attrtype><attwidth Sync="TRUE">19</attwidth><atprecis Sync="TRUE">0</atprecis><attscale Sync="TRUE">0</attscale><attrdef Sync="TRUE">Area of feature in internal units squared.</attrdef><attrdefs Sync="TRUE">Esri</attrdefs><attrdomv><udom Sync="TRUE">Positive real numbers that are automatically generated.</udom></attrdomv></attr></detailed></eainfo><mdDateSt Sync="TRUE">20210316</mdDateSt><mdChar><CharSetCd value="004" Sync="TRUE"></CharSetCd></mdChar></metadata>  
@@ -21,11 +21,11 @@ class Api(ApiTemplate): @@ -21,11 +21,11 @@ class Api(ApiTemplate):
21 raise Exception("数据不存在！") 21 raise Exception("数据不存在！")
22 pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) 22 pg_ds = PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri))
23 layer:Layer = pg_ds.GetLayerByName(table.name) 23 layer:Layer = pg_ds.GetLayerByName(table.name)
24 - append_dict ={"epsg":None,"sr_wkt":None,"sr_proj4":None,"exist":1} 24 + append_dict ={"srid":None,"sr_wkt":None,"sr_proj4":None,"exist":1}
25 if layer: 25 if layer:
26 sr:SpatialReference = layer.GetSpatialRef() 26 sr:SpatialReference = layer.GetSpatialRef()
27 if sr: 27 if sr:
28 - append_dict["epsg"] = sr.GetAuthorityCode(None) 28 + append_dict["srid"] = sr.GetAuthorityCode(None)
29 append_dict["sr_wkt"] = sr.ExportToWkt() 29 append_dict["sr_wkt"] = sr.ExportToWkt()
30 append_dict["sr_proj4"] = sr.ExportToProj4() 30 append_dict["sr_proj4"] = sr.ExportToProj4()
31 else: 31 else:
@@ -47,7 +47,8 @@ class Api(ApiTemplate): @@ -47,7 +47,8 @@ class Api(ApiTemplate):
47 creator=self.para.get("creator"), 47 creator=self.para.get("creator"),
48 file_name=None, 48 file_name=None,
49 database_guid=database.guid, 49 database_guid=database.guid,
50 - process="数据库更新中") 50 + process="数据库更新中",
  51 + task_pid=refresh_process.pid)
51 52
52 db.session.add(task) 53 db.session.add(task)
53 db.session.commit() 54 db.session.commit()
@@ -482,10 +483,6 @@ class Api(ApiTemplate): @@ -482,10 +483,6 @@ class Api(ApiTemplate):
482 StructurePrint().print("{}表要素属性减少！".format(table_name)) 483 StructurePrint().print("{}表要素属性减少！".format(table_name))
483 sys_session.delete(column) 484 sys_session.delete(column)
484 485
485 - # 修改要素量  
486 - # sql = 'select count(*) from "{}"'.format(table_name)  
487 - # count = data_session.execute(sql).fetchone()[0]  
488 -  
489 count = SQLUtil.get_table_count(table_name,data_session) 486 count = SQLUtil.get_table_count(table_name,data_session)
490 487
491 if not table.feature_count.__eq__(count): 488 if not table.feature_count.__eq__(count):
@@ -70,9 +70,6 @@ class Api(ApiTemplate): @@ -70,9 +70,6 @@ class Api(ApiTemplate):
70 vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid)) 70 vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid))
71 vacuate_process.start() 71 vacuate_process.start()
72 72
73 - # ref_api = RefApi()  
74 - # ref_api.para["guid"] = table_guid  
75 - # ref_grids = ref_api.process()["data"]  
76 73
77 task = Task(guid=task_guid, 74 task = Task(guid=task_guid,
78 name="{}精化".format(table.name), 75 name="{}精化".format(table.name),
@@ -84,6 +81,7 @@ class Api(ApiTemplate): @@ -84,6 +81,7 @@ class Api(ApiTemplate):
84 file_name=None, 81 file_name=None,
85 database_guid=table.database_guid, 82 database_guid=table.database_guid,
86 process="精化中", 83 process="精化中",
  84 + task_pid= vacuate_process.pid
87 # parameter=",".join([str(x) for x in ref_grids]) 85 # parameter=",".join([str(x) for x in ref_grids])
88 ) 86 )
89 87
@@ -136,12 +134,15 @@ class Api(ApiTemplate): @@ -136,12 +134,15 @@ class Api(ApiTemplate):
136 134
137 vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri) 135 vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri)
138 136
139 - 137 + count = 0
140 for feature in layer: 138 for feature in layer:
141 geo = feature.GetGeometryRef() 139 geo = feature.GetGeometryRef()
142 #插入抽稀图层 140 #插入抽稀图层
143 if geo is not None: 141 if geo is not None:
144 vacuate_process.vacuate(geo,feature) 142 vacuate_process.vacuate(geo,feature)
  143 + count += 1
  144 + if count%10000==0:
  145 + StructurePrint().print("{}图层已抽稀{}个对象".format(table.name, count))
145 146
146 vacuate_process.set_vacuate_count() 147 vacuate_process.set_vacuate_count()
147 148
@@ -297,12 +298,6 @@ class VacuateProcess: @@ -297,12 +298,6 @@ class VacuateProcess:
297 layer.ResetReading() 298 layer.ResetReading()
298 299
299 300
300 -  
301 - # 额外一层  
302 - # self.this_gridsize.append(0.000075)  
303 - # self.max_level += 1  
304 - ######  
305 -  
306 if extent[0]>180: 301 if extent[0]>180:
307 self.t_grid_size=self.project_gridsize 302 self.t_grid_size=self.project_gridsize
308 else: 303 else:
@@ -324,7 +319,6 @@ class VacuateProcess: @@ -324,7 +319,6 @@ class VacuateProcess:
324 319
325 # 创建抽稀ds 320 # 创建抽稀ds
326 for l in range(self.max_level): 321 for l in range(self.max_level):
327 - # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri))  
328 if configure.VACUATE_DB_URI: 322 if configure.VACUATE_DB_URI:
329 pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI) 323 pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.VACUATE_DB_URI)
330 else: 324 else:
@@ -387,13 +381,6 @@ class VacuateProcess: @@ -387,13 +381,6 @@ class VacuateProcess:
387 lat_extent = extent[3]-extent[2] 381 lat_extent = extent[3]-extent[2]
388 382
389 this_grid_len =self.vacuate_layers_gridsize[level] 383 this_grid_len =self.vacuate_layers_gridsize[level]
390 - #超大的直接加入  
391 - # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len:  
392 - # vacuate_layer: Layer = self.vacuate_layers.get(level)  
393 - # feat = ogr.Feature(vacuate_layer.GetLayerDefn())  
394 - # feat.SetGeometry(g)  
395 - # vacuate_layer.CreateFeature(feat)  
396 - # else:  
397 384
398 row = int((center.GetY() - self.extent[2]) / this_grid_len) 385 row = int((center.GetY() - self.extent[2]) / this_grid_len)
399 col = int((center.GetX() - self.extent[0]) / this_grid_len) 386 col = int((center.GetX() - self.extent[0]) / this_grid_len)
@@ -420,7 +407,6 @@ class VacuateProcess: @@ -420,7 +407,6 @@ class VacuateProcess:
420 vacuate_layer.CreateFeature(feat) 407 vacuate_layer.CreateFeature(feat)
421 self.fill_dict[key] += 1 408 self.fill_dict[key] += 1
422 #超大的还有机会 409 #超大的还有机会
423 - # elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:  
424 elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len: 410 elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len:
425 vacuate_layer: Layer = self.vacuate_layers.get(level) 411 vacuate_layer: Layer = self.vacuate_layers.get(level)
426 feat = ogr.Feature(vacuate_layer.GetLayerDefn()) 412 feat = ogr.Feature(vacuate_layer.GetLayerDefn())
@@ -9,6 +9,7 @@ from ..models import Table, Database, DES,Task,db,TableVacuate @@ -9,6 +9,7 @@ from ..models import Table, Database, DES,Task,db,TableVacuate
9 from app.util.component.ApiTemplate import ApiTemplate 9 from app.util.component.ApiTemplate import ApiTemplate
10 from app.util.component.PGUtil import PGUtil 10 from app.util.component.PGUtil import PGUtil
11 from app.util.component.EntryDataVacuate import Process 11 from app.util.component.EntryDataVacuate import Process
  12 +from app.util.component.StructuredPrint import StructurePrint
12 import multiprocessing 13 import multiprocessing
13 import uuid 14 import uuid
14 import configure 15 import configure
@@ -26,7 +27,7 @@ class Api(ApiTemplate): @@ -26,7 +27,7 @@ class Api(ApiTemplate):
26 db_session = None 27 db_session = None
27 try: 28 try:
28 table_guid = self.para.get("guid") 29 table_guid = self.para.get("guid")
29 - grids = [] 30 +
30 if not self.para.get("grids"): 31 if not self.para.get("grids"):
31 raise Exception("请输入grids参数！") 32 raise Exception("请输入grids参数！")
32 else: 33 else:
@@ -54,7 +55,6 @@ class Api(ApiTemplate): @@ -54,7 +55,6 @@ class Api(ApiTemplate):
54 pg_ds.Destroy() 55 pg_ds.Destroy()
55 56
56 57
57 -  
58 if Task.query.filter_by(table_guid=table_guid,state=0).one_or_none(): 58 if Task.query.filter_by(table_guid=table_guid,state=0).one_or_none():
59 res["result"] = False 59 res["result"] = False
60 res["msg"] = "数据精化中！" 60 res["msg"] = "数据精化中！"
@@ -64,11 +64,6 @@ class Api(ApiTemplate): @@ -64,11 +64,6 @@ class Api(ApiTemplate):
64 res["msg"] = "非空间表！" 64 res["msg"] = "非空间表！"
65 return res 65 return res
66 66
67 - # if table.is_vacuate==1:  
68 - # res["state"] = -1  
69 - # res["message"] = "已精化！"  
70 - # return res  
71 -  
72 # 初始化task 67 # 初始化task
73 task_guid = uuid.uuid1().__str__() 68 task_guid = uuid.uuid1().__str__()
74 69
@@ -86,7 +81,8 @@ class Api(ApiTemplate): @@ -86,7 +81,8 @@ class Api(ApiTemplate):
86 file_name=None, 81 file_name=None,
87 database_guid=table.database_guid, 82 database_guid=table.database_guid,
88 process="精化中", 83 process="精化中",
89 - parameter=self.para.get("grids")) 84 + parameter=self.para.get("grids"),
  85 + task_pid=vacuate_process.pid)
90 86
91 db.session.add(task) 87 db.session.add(task)
92 db.session.commit() 88 db.session.commit()
@@ -140,12 +136,15 @@ class Api(ApiTemplate): @@ -140,12 +136,15 @@ class Api(ApiTemplate):
140 136
141 vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri,grids) 137 vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri,grids)
142 138
143 - 139 + count = 0
144 for feature in layer: 140 for feature in layer:
145 geo = feature.GetGeometryRef() 141 geo = feature.GetGeometryRef()
146 #插入抽稀图层 142 #插入抽稀图层
147 if geo is not None: 143 if geo is not None:
148 vacuate_process.vacuate(geo,feature) 144 vacuate_process.vacuate(geo,feature)
  145 + count += 1
  146 + if count%10000==0:
  147 + StructurePrint().print("{}图层已抽稀{}个对象".format(table.name, count))
149 148
150 vacuate_process.set_vacuate_count() 149 vacuate_process.set_vacuate_count()
151 150
@@ -346,13 +345,7 @@ class VacuateProcess: @@ -346,13 +345,7 @@ class VacuateProcess:
346 lat_extent = extent[3]-extent[2] 345 lat_extent = extent[3]-extent[2]
347 346
348 this_grid_len =self.vacuate_layers_gridsize[level] 347 this_grid_len =self.vacuate_layers_gridsize[level]
349 - #超大的直接加入  
350 - # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len:  
351 - # vacuate_layer: Layer = self.vacuate_layers.get(level)  
352 - # feat = ogr.Feature(vacuate_layer.GetLayerDefn())  
353 - # feat.SetGeometry(g)  
354 - # vacuate_layer.CreateFeature(feat)  
355 - # else: 348 +
356 349
357 row = int((center.GetY() - self.extent[2]) / this_grid_len) 350 row = int((center.GetY() - self.extent[2]) / this_grid_len)
358 col = int((center.GetX() - self.extent[0]) / this_grid_len) 351 col = int((center.GetX() - self.extent[0]) / this_grid_len)
@@ -380,7 +373,6 @@ class VacuateProcess: @@ -380,7 +373,6 @@ class VacuateProcess:
380 self.fill_dict[key] += 1 373 self.fill_dict[key] += 1
381 374
382 #超大的还有机会 375 #超大的还有机会
383 - # elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:  
384 elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len: 376 elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len:
385 vacuate_layer: Layer = self.vacuate_layers.get(level) 377 vacuate_layer: Layer = self.vacuate_layers.get(level)
386 feat = ogr.Feature(vacuate_layer.GetLayerDefn()) 378 feat = ogr.Feature(vacuate_layer.GetLayerDefn())
@@ -10,6 +10,7 @@ from . import task_list @@ -10,6 +10,7 @@ from . import task_list
10 from . import task_detail 10 from . import task_detail
11 from . import task_delete 11 from . import task_delete
12 from . import task_count 12 from . import task_count
  13 +from . import task_kill
13 14
14 class DataManager(BlueprintApi): 15 class DataManager(BlueprintApi):
15 16
@@ -44,6 +45,15 @@ class DataManager(BlueprintApi): @@ -44,6 +45,15 @@ class DataManager(BlueprintApi):
44 return task_delete.Api().result 45 return task_delete.Api().result
45 46
46 @staticmethod 47 @staticmethod
  48 + @bp.route('/Kill', methods=['POST'])
  49 + @swag_from(task_kill.Api.api_doc)
  50 + def task_kill():
  51 + """
  52 + Kill任务
  53 + """
  54 + return task_kill.Api().result
  55 +
  56 + @staticmethod
47 @bp.route('/Count', methods=['POST']) 57 @bp.route('/Count', methods=['POST'])
48 @swag_from(task_count.Api.api_doc) 58 @swag_from(task_count.Api.api_doc)
49 def task_count(): 59 def task_count():
@@ -52,8 +52,6 @@ def task_consumer(): @@ -52,8 +52,6 @@ def task_consumer():
52 # 已经结束的进程 从监测中删除 52 # 已经结束的进程 从监测中删除
53 remove_process = [] 53 remove_process = []
54 54
55 - # structured_print(running_dict.__len__().__str__())  
56 -  
57 for process, layer_names in running_dict.items(): 55 for process, layer_names in running_dict.items():
58 if not process.is_alive(): 56 if not process.is_alive():
59 for l in layer_names: 57 for l in layer_names:
@@ -66,7 +64,6 @@ def task_consumer(): @@ -66,7 +64,6 @@ def task_consumer():
66 for process in remove_process: 64 for process in remove_process:
67 running_dict.pop(process) 65 running_dict.pop(process)
68 66
69 - # StructurePrint().print("listening...")  
70 67
71 # 入库进程少于阈值，开启入库进程 68 # 入库进程少于阈值，开启入库进程
72 69
@@ -221,10 +218,6 @@ def download_gdb( sys_session, table_names, ds, database_guid): @@ -221,10 +218,6 @@ def download_gdb( sys_session, table_names, ds, database_guid):
221 218
222 table_alias = table.alias 219 table_alias = table.alias
223 220
224 - # if is_chinese(table_name):  
225 - # if not table_alias:  
226 - # table_alias = table_name  
227 - # table_name = "table{}".format(table_name.__hash__())  
228 221
229 fid = layer.GetFIDColumn() 222 fid = layer.GetFIDColumn()
230 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(), 223 pg_layer: Layer = gdb_ds.CreateLayer(table_name, layer.GetSpatialRef(), layer.GetGeomType(),
@@ -244,7 +237,7 @@ def download_gdb( sys_session, table_names, ds, database_guid): @@ -244,7 +237,7 @@ def download_gdb( sys_session, table_names, ds, database_guid):
244 feature.SetFID(feature.GetFID() + offset) 237 feature.SetFID(feature.GetFID() + offset)
245 pg_layer.CreateFeature(feature) 238 pg_layer.CreateFeature(feature)
246 239
247 - # gdb_ds.CopyLayer(layer, table_name,["LAYER_ALIAS={}".format(table_alias)]) 240 +
248 241
249 gdb_ds.Destroy() 242 gdb_ds.Destroy()
250 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path]) 243 ZipUtil.create_zip(gdb_path + ".zip", [gdb_path])
@@ -760,10 +753,7 @@ def task_vacuate(table,task_guid): @@ -760,10 +753,7 @@ def task_vacuate(table,task_guid):
760 tvs = sys_session.query(TableVacuate).filter_by(table_guid=table.guid).all() 753 tvs = sys_session.query(TableVacuate).filter_by(table_guid=table.guid).all()
761 for tv in tvs: 754 for tv in tvs:
762 sys_session.delete(tv) 755 sys_session.delete(tv)
763 - # try:  
764 - # pg_ds.DeleteLayer(tv.name)  
765 - # except Exception as e :  
766 - # StructurePrint().print("抽稀图层不存在！","warn") 756 +
767 757
768 # 创建抽稀过程 758 # 创建抽稀过程
769 options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name, pg_session)), 759 options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name, pg_session)),
@@ -3,14 +3,15 @@ @@ -3,14 +3,15 @@
3 #createtime: 2020/9/4 3 #createtime: 2020/9/4
4 #email: nheweijun@sina.com 4 #email: nheweijun@sina.com
5 5
6 -from ..models import db,Task,Table  
7 - 6 +from ..models import db,Task,Table,InsertingLayerName
8 7
  8 +from app.modules.service.models import Image
9 from app.util.component.ApiTemplate import ApiTemplate 9 from app.util.component.ApiTemplate import ApiTemplate
  10 +from app.util.component.StructuredPrint import StructurePrint
10 import os 11 import os
11 import signal 12 import signal
12 import platform 13 import platform
13 - 14 +import json
14 15
15 class Api(ApiTemplate): 16 class Api(ApiTemplate):
16 api_name = "停止任务" 17 api_name = "停止任务"
@@ -23,12 +24,18 @@ class Api(ApiTemplate): @@ -23,12 +24,18 @@ class Api(ApiTemplate):
23 task_guid = self.para.get("task_guid") 24 task_guid = self.para.get("task_guid")
24 task = Task.query.filter_by(guid=task_guid).one_or_none() 25 task = Task.query.filter_by(guid=task_guid).one_or_none()
25 pid = task.task_pid 26 pid = task.task_pid
26 - if platform.system().lower().__contains__("windows"):  
27 - os.popen('taskkill.exe /pid:' + str(pid))  
28 - else:  
29 - os.kill(pid,signal.SIGILL) 27 + try:
  28 + if platform.system().lower().__contains__("windows"):
  29 + exec_result = os.popen('taskkill.exe /pid:' + str(pid))
  30 +
  31 + else:
  32 + os.kill(pid,signal.SIGILL)
  33 + except Exception as e:
  34 + StructurePrint.print("Kill task 失败")
  35 + raise e
  36 +
30 #处理kill任务后的事情 37 #处理kill任务后的事情
31 - self.fix_task() 38 + self.fix_task(task)
32 res["msg"] = "Kill成功！" 39 res["msg"] = "Kill成功！"
33 res["result"] = True 40 res["result"] = True
34 except Exception as e: 41 except Exception as e:
@@ -38,18 +45,29 @@ class Api(ApiTemplate): @@ -38,18 +45,29 @@ class Api(ApiTemplate):
38 45
39 46
40 def fix_task(self,task): 47 def fix_task(self,task):
  48 +
41 if task.task_type==1: 49 if task.task_type==1:
42 - pass 50 + ilns = InsertingLayerName.query.filter_by(task_guid=task.guid).all()
  51 + for iln in ilns:
  52 + db.session.delete(iln)
  53 +
43 if task.task_type==2: 54 if task.task_type==2:
44 table = Table.query.filter_by(guid=task.table_guid).one_or_none() 55 table = Table.query.filter_by(guid=task.table_guid).one_or_none()
45 if len(table.relate_table_vacuates.all())>0: 56 if len(table.relate_table_vacuates.all())>0:
46 Table.query.filter_by(guid=task.table_guid).update({"is_vacuate":1}) 57 Table.query.filter_by(guid=task.table_guid).update({"is_vacuate":1})
  58 + else:
  59 + Table.query.filter_by(guid=task.table_guid).update({"is_vacuate": 0})
  60 +
47 if task.task_type==3: 61 if task.task_type==3:
48 pass 62 pass
49 - if task.task_type==3: 63 + if task.task_type==4:
50 pass 64 pass
  65 + if task.task_type==5:
  66 + Image.query.filter_by(guid=task.parameter).update({"has_pyramid":0})
51 67
  68 + Task.query.filter_by(guid=task.guid).update({"state":-1})
52 db.session.commit() 69 db.session.commit()
  70 + return None
53 71
54 api_doc = { 72 api_doc = {
55 "tags": ["任务接口"], 73 "tags": ["任务接口"],
@@ -9,6 +9,9 @@ import datetime @@ -9,6 +9,9 @@ import datetime
9 from ..models import Task 9 from ..models import Task
10 from app.util.component.ApiTemplate import ApiTemplate 10 from app.util.component.ApiTemplate import ApiTemplate
11 from app.util.component.ModelVisitor import ModelVisitor 11 from app.util.component.ModelVisitor import ModelVisitor
  12 +import json
  13 +
  14 +
12 class Api(ApiTemplate): 15 class Api(ApiTemplate):
13 api_name = "任务列表" 16 api_name = "任务列表"
14 def para_check(self): 17 def para_check(self):
@@ -47,6 +50,19 @@ class Api(ApiTemplate): @@ -47,6 +50,19 @@ class Api(ApiTemplate):
47 tasks = tasks.limit(page_size).offset(page_index * page_size).all() 50 tasks = tasks.limit(page_size).offset(page_index * page_size).all()
48 51
49 res["data"]["list"] = [ModelVisitor.task_to_json(task) for task in tasks] 52 res["data"]["list"] = [ModelVisitor.task_to_json(task) for task in tasks]
  53 +
  54 + try:
  55 + for task_info in res["data"]["list"]:
  56 + if task_info["task_type"]==1:
  57 + task_info["layers"] = []
  58 + meta = json.loads(json.loads(task_info.get("parameter")).get("meta"))
  59 + for m in meta:
  60 + for key in m.get("layer").keys():
  61 +
  62 + task_info["layers"].append({"name":key,"new_name":m.get("layer").get(key)})
  63 + except:
  64 + pass
  65 +
50 res["result"] = True 66 res["result"] = True
51 except Exception as e: 67 except Exception as e:
52 raise e 68 raise e
1 -# coding=utf-8  
2 -# author: 4N  
3 -#createtime: 2021/5/18  
4 -#email: nheweijun@sina.com  
5 -  
6 -  
7 -from flasgger import swag_from  
8 -from flask import Blueprint  
9 -from app.util import BlueprintApi  
10 -  
11 -from . import monitor_info, monitoring  
12 -  
13 -  
14 -class Monitor(BlueprintApi):  
15 -  
16 - bp = Blueprint("Monitor", __name__, url_prefix="/API/Monitor")  
17 -  
18 - @staticmethod  
19 - @bp.route('/Info', methods=['GET'])  
20 - @swag_from(monitor_info.Api.api_doc)  
21 - def monitor_info():  
22 - """  
23 - 性能监控  
24 - """  
25 - return monitor_info.Api().result  
26 -  
27 - @staticmethod  
28 - @bp.route('/baseMonitoring', methods=['GET'])  
29 - @swag_from(monitoring.Api.api_doc)  
30 - def monitoring():  
31 - """  
32 - 基础监控  
33 - """  
34 - return monitoring.Api().result  
1 -# coding=utf-8  
2 -#author: 4N  
3 -#createtime: 2021/6/11  
4 -#email: nheweijun@sina.com  
5 -  
6 -  
7 -from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Time  
8 -from app.models import db  
9 -  
1 -# coding=utf-8  
2 -#author: 4N  
3 -#createtime: 2021/7/9  
4 -#email: nheweijun@sina.com  
5 -  
6 -from app.models import *  
7 -  
8 -from app.util.component.ApiTemplate import ApiTemplate  
9 -from app.util.component.ModelVisitor import ModelVisitor  
10 -import psutil  
11 -  
12 -class Api(ApiTemplate):  
13 - api_name = "监控"  
14 -  
15 - def process(self):  
16 -  
17 - # 返回结果  
18 - res = {}  
19 - res["data"] = {}  
20 - try:  
21 - # 业务逻辑  
22 - cpu_count = psutil.cpu_count(False)  
23 - cpu_per = int(psutil.cpu_percent())  
24 - res["data"]["cpu"] ={"count":cpu_count,"percent":"{}%".format(cpu_per)}  
25 -  
26 -  
27 - mem_total = int(psutil.virtual_memory()[0])  
28 - mem_used = int(psutil.virtual_memory()[3])  
29 - mem_per = int(psutil.virtual_memory()[2])  
30 -  
31 - res["data"]["memory"] = {  
32 - 'total': self.format_value(mem_total),  
33 - 'used': self.format_value(mem_used),  
34 - 'percent': "{}%".format(mem_per)  
35 - }  
36 -  
37 -  
38 -  
39 - network_sent = int(psutil.net_io_counters()[0] / 8 ) # 每秒接受的kb  
40 - network_recv = int(psutil.net_io_counters()[1] / 8 )  
41 -  
42 - res["data"]["network"] = {  
43 - 'sent': self.format_value(network_sent),  
44 - 'recv': self.format_value(network_recv)  
45 - }  
46 -  
47 -  
48 - res["result"] = True  
49 - except Exception as e:  
50 - raise e  
51 - return res  
52 -  
53 - api_doc = {  
54 -  
55 - "tags": ["监控接口"],  
56 - "parameters": [  
57 - ],  
58 - "responses": {  
59 - 200: {  
60 - "schema": {  
61 - "properties": {  
62 - }  
63 - }  
64 - }  
65 - }  
66 - }  
67 -  
68 - def format_value(self,value):  
69 - if value>1024**3:  
70 - value = "{}GB".format(format(value/1024.0**3,'.1f'))  
71 - elif value>1024**2:  
72 - value = "{}MB".format(format(value / 1024.0 ** 2, '.1f'))  
73 - elif value>1024:  
74 - value = "{}KB".format(format(value / 1024.0, '.1f'))  
75 - else:  
76 - value = "{}B".format(format(value, '.1f'))  
77 - return value  
78 -  
79 -if __name__ == '__main__':  
80 - api = Api()  
81 - api.process()  
1 -from app.models import *  
2 -from app.util.component.ApiTemplate import ApiTemplate  
3 -import paramiko  
4 -import time  
5 -  
6 -  
7 -class Api(ApiTemplate):  
8 - api_name = "远程监控"  
9 -  
10 - def process(self):  
11 -  
12 - # 返回结果  
13 - res = {}  
14 - res["data"] = {}  
15 - try:  
16 - # 业务逻辑  
17 - client = paramiko.SSHClient()  
18 - client.set_missing_host_key_policy(paramiko.AutoAddPolicy)  
19 - client.connect(hostname='172.26.60.100',  
20 - username='root', password='DMap@123')  
21 - # cpu  
22 - order = "top -b -n1 | sed -n '3p' | awk '{print $2}'"  
23 - stdin, stdout, stderr = client.exec_command(order)  
24 - cpu_usage = stdout.read().decode().split("\n")[0] # cpu使用率  
25 -  
26 - # 内存  
27 - order = "free -h | sed -n '2p' | awk '{print $2}'"  
28 - stdin, stdout, stderr = client.exec_command(order)  
29 - totalMem = stdout.read().decode().split("\n")[0] # 总内存  
30 -  
31 - order = "free -h | sed -n '3p' | awk '{print $3}'"  
32 - stdin, stdout, stderr = client.exec_command(order)  
33 - freeMem = stdout.read().decode().split("\n")[0] # 空余内存  
34 -  
35 - # disk  
36 - order = "df -m | grep -v 'overlay\|Filesystem' | awk '{print $1,$2,$3}' | grep /dev | awk '{print $2}' | awk -v total=0 '{total+=$1}END{print total}'"  
37 - stdin, stdout, stderr = client.exec_command(order)  
38 - totalDisk = int(stdout.read().decode().split("\n")  
39 - [0]) # 总磁盘空间，单位Mb  
40 -  
41 - order = "df -m | grep -v 'overlay\|Filesystem' | awk '{print $1,$2,$3}' | grep /dev | awk '{print $3}' | awk -v total=0 '{total+=$1}END{print total}'"  
42 - stdin, stdout, stderr = client.exec_command(order)  
43 - usedDisk = int(stdout.read().decode().split("\n")  
44 - [0]) # 已使用磁盘空间，单位Mb  
45 -  
46 - # network  
47 - # 接收的字节数  
48 - rx_time = []  
49 - rx_bytes = []  
50 - tx_time = []  
51 - tx_bytes = []  
52 -  
53 - # 接收的字节数  
54 - order = "ifconfig | grep RX | grep -v 'errors'| awk -v total=0 '{total+=$5}END{print total}'"  
55 - i = 0  
56 - while i < 2:  
57 - i = i+1  
58 - stdin, stdout, stderr = client.exec_command(order)  
59 - rx_time.append(time.time())  
60 - rx_bytes.append(int(stdout.read().decode().split("\n")[0]))  
61 -  
62 - # 发送的字节数  
63 - order = "ifconfig | grep TX | grep -v 'errors'| awk -v total=0 '{total+=$5}END{print total}'"  
64 - i = 0  
65 - while i < 2:  
66 - i = i+1  
67 - stdin, stdout, stderr = client.exec_command(order)  
68 - tx_time.append(time.time())  
69 - tx_bytes.append(int(stdout.read().decode().split("\n")[0]))  
70 -  
71 - res["data"] = {  
72 - "cpuUsage": "{}%".format(cpu_usage),  
73 - "totalMemory": "{}".format(totalMem),  
74 - "freeMemory": "{}".format(freeMem),  
75 - "totalDisk": "{}".format(self.format_value(totalDisk*1024**2)),  
76 - "usedDisk": "{}".format(self.format_value(usedDisk*1024**2)),  
77 - "networkRecv": "{}".format(self.format_value((rx_bytes[1] - rx_bytes[0])/(rx_time[1]-rx_time[0]))),  
78 - "networkSend": "{}".format(self.format_value((tx_bytes[1] - tx_bytes[0])/(tx_time[1]-tx_time[0])))  
79 - }  
80 -  
81 - res["result"] = True  
82 - except Exception as e:  
83 - raise e  
84 - finally:  
85 - client.close()  
86 - return res  
87 -  
88 - api_doc = {  
89 -  
90 - "tags": ["监控接口"],  
91 - "parameters": [  
92 - ],  
93 - "responses": {  
94 - 200: {  
95 - "schema": {  
96 - "properties": {  
97 - }  
98 - }  
99 - }  
100 - }  
101 - }  
102 -  
103 - def format_value(self, value):  
104 - if value > 1024**3:  
105 - value = "{}GB".format(format(value/1024.0**3, '.1f'))  
106 - elif value > 1024**2:  
107 - value = "{}MB".format(format(value / 1024.0 ** 2, '.1f'))  
108 - elif value > 1024:  
109 - value = "{}KB".format(format(value / 1024.0, '.1f'))  
110 - else:  
111 - value = "{}B".format(format(value, '.1f'))  
112 - return value  
113 -  
114 -  
115 -if __name__ == '__main__':  
116 - api = Api()  
117 - api.process()  
@@ -20,7 +20,8 @@ from . import image_register,image_list,image_info,image_edit,image_overview @@ -20,7 +20,8 @@ from . import image_register,image_list,image_info,image_edit,image_overview
20 from . import image_tag_create,image_tag_delete,image_tag_list 20 from . import image_tag_create,image_tag_delete,image_tag_list
21 from . import image_wms_temporary 21 from . import image_wms_temporary
22 from . import image_wms_kv 22 from . import image_wms_kv
23 - 23 +from . import image_build_pyramid
  24 +from . import image_refresh
24 25
25 class ImageServerInstance: 26 class ImageServerInstance:
26 pass 27 pass
@@ -86,6 +87,24 @@ class ImageManager(BlueprintApi): @@ -86,6 +87,24 @@ class ImageManager(BlueprintApi):
86 """ 87 """
87 return image_overview.Api().result 88 return image_overview.Api().result
88 89
  90 + @staticmethod
  91 + @bp.route('/BuildPyramid', methods=['POST'])
  92 + @swag_from(image_build_pyramid.Api.api_doc)
  93 + def api_image_build_pyramid():
  94 + """
  95 + 创建影像金字塔
  96 + """
  97 + return image_build_pyramid.Api().result
  98 +
  99 + @staticmethod
  100 + @bp.route('/Refresh', methods=['POST'])
  101 + @swag_from(image_refresh.Api.api_doc)
  102 + def api_image_refresh():
  103 + """
  104 + 影像刷新信息
  105 + """
  106 + return image_refresh.Api().result
  107 +
89 108
90 @staticmethod 109 @staticmethod
91 @bp.route('/TagCreate', methods=['POST']) 110 @bp.route('/TagCreate', methods=['POST'])
1 -# coding=utf-8  
2 -#author: 4N  
3 -#createtime: 2021/10/18  
4 -#email: nheweijun@sina.com  
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/7/19
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +from app.util.component.ApiTemplate import ApiTemplate
  8 +import datetime
  9 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
  10 +from app.models import db
  11 +from app.modules.service.models import Image
  12 +from app.modules.data.models import Task
  13 +import multiprocessing
  14 +import uuid
  15 +import configure
  16 +from app.util.component.PGUtil import PGUtil
  17 +from .util.Cache import Cache
  18 +import json
  19 +from osgeo import gdal,osr
  20 +from osgeo.gdal import *
  21 +import traceback
  22 +import os
  23 +
  24 +class Api(ApiTemplate):
  25 +
  26 + api_name = "创建影像金字塔"
  27 +
  28 + def process(self):
  29 +
  30 + # 返回结果
  31 + res = {}
  32 + try:
  33 + image_guid = self.para.get("guid")
  34 + task_guid = uuid.uuid1().__str__()
  35 + image = Image.query.filter_by(guid=image_guid).one_or_none()
  36 + if not image:
  37 + raise Exception("数据不存在！")
  38 + if image.has_pyramid==-1:
  39 + raise Exception("数据正在创建金字塔！")
  40 +
  41 + image_service_info, zoo, servers = Cache.cache_data(None)
  42 +
  43 + image_servers = image.server.split(",")
  44 + image_servers = [ser for ser in image_servers if ser in servers]
  45 +
  46 + if len(image_servers)==0:
  47 + raise Exception("暂时没有可用数据服务器提供该影像相关服务！")
  48 +
  49 + #创建金字塔的进程
  50 + build_process = multiprocessing.Process(target=self.build_pyramid_task,
  51 + args=(image_guid,task_guid,image_servers,image.path))
  52 + build_process.start()
  53 +
  54 + task = Task(guid=task_guid,
  55 + name="{}创建金字塔".format(image.name),
  56 + create_time=datetime.datetime.now(),
  57 + state=0,
  58 + task_type=5,
  59 + creator=self.para.get("creator"),
  60 + process="创建金字塔",
  61 + task_pid=build_process.pid,
  62 + parameter=image_guid)
  63 +
  64 + db.session.add(task)
  65 + db.session.commit()
  66 +
  67 +
  68 + res["data"] = "创建金字塔任务已提交!"
  69 +
  70 + res["result"] = True
  71 +
  72 + except Exception as e:
  73 + raise e
  74 +
  75 + return res
  76 +
  77 + def build_pyramid_task(self,image_guid,task_guid,data_servers,path):
  78 + sys_session = None
  79 + try:
  80 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
  81 + #进入创建金字塔的状态
  82 + sys_session.query(Image).filter_by(guid=image_guid).update({"has_pyramid": -1})
  83 + sys_session.query(Task).filter_by(guid=task_guid).update({"state": 2})
  84 +
  85 + sys_session.commit()
  86 + sys_session.close()
  87 +
  88 +
  89 + #所有数据节点的影像都要建金字塔
  90 + update_size = None
  91 + overview_count = None
  92 + for data_server in data_servers:
  93 + if data_server=="本地服务器":
  94 + result = self.buildOverview(path)
  95 + update_size = os.path.getsize(path)
  96 + else:
  97 + thrift_connect = ThriftConnect(data_server)
  98 + image_info = json.loads(thrift_connect.client.getInfo(path))
  99 + if image_info.get("overview_count") > 0:
  100 + overview_count = image_info.get("overview_count")
  101 + continue
  102 + result = thrift_connect.client.buildOverview(path)
  103 + update_size = json.loads(thrift_connect.client.getInfo(path)).get("size")
  104 + thrift_connect.close()
  105 +
  106 + #重新连接，防止session超时
  107 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
  108 +
  109 + sys_session.query(Task).filter_by(guid=task_guid).update({"state":1,"update_time":datetime.datetime.now()})
  110 + image:Image = sys_session.query(Image).filter_by(guid=image_guid).one_or_none()
  111 +
  112 + if not overview_count:
  113 + overview_count = 0
  114 + raster_x_size = image.raster_x_size
  115 + raster_y_size = image.raster_y_size
  116 + while raster_x_size * raster_y_size > 65536:
  117 + raster_x_size /= 2.0
  118 + raster_y_size /= 2.0
  119 + overview_count += 1
  120 +
  121 + image.has_pyramid = 1
  122 + image.overview_count = overview_count
  123 + if update_size:
  124 + image.size = update_size
  125 +
  126 + except:
  127 + sys_session.query(Image).filter_by(guid=image_guid).update({"has_pyramid": 0})
  128 + sys_session.query(Task).filter_by(guid=task_guid).update({"state": -1,"update_time":datetime.datetime.now()})
  129 + finally:
  130 + sys_session.commit()
  131 + if sys_session:
  132 + try:
  133 + sys_session.close()
  134 + except:
  135 + pass
  136 +
  137 +
  138 +
  139 +
  140 + def buildOverview(self,path):
  141 + image: Dataset = gdal.Open(path, 1)
  142 + result =True
  143 + try:
  144 + overviews=[]
  145 + raster_x_size = image.RasterXSize
  146 + raster_y_size = image.RasterYSize
  147 + while raster_x_size*raster_y_size>65536:
  148 + raster_x_size /= 2.0
  149 + raster_y_size /= 2.0
  150 + overviews.append(round(image.RasterXSize/raster_x_size))
  151 + band: Band = image.GetRasterBand(1)
  152 + if len(overviews) == band.GetOverviewCount():
  153 + pass
  154 + else:
  155 + image.BuildOverviews("AVERAGE",overviews)
  156 + except Exception as e:
  157 + print(traceback.format_exc())
  158 + result = False
  159 + finally:
  160 + del image
  161 + return result
  162 +
  163 + api_doc = {
  164 + "tags": ["影像接口"],
  165 + "parameters": [
  166 + {"name": "guid",
  167 + "in": "formData",
  168 + "type": "string",
  169 + "description": "guid"},
  170 + ],
  171 + "responses": {
  172 + 200: {
  173 + "schema": {
  174 + "properties": {
  175 + }
  176 + }
  177 + }
  178 + }
  179 + }
  180 +
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/7/19
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +from osgeo import gdal,osr
  8 +from osgeo.gdal import Dataset,Band
  9 +from app.util.component.ApiTemplate import ApiTemplate
  10 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
  11 +import json
  12 +from ..models import Image
  13 +import datetime
  14 +from app.models import db
  15 +import uuid
  16 +import os
  17 +from .util.ImageType import ImageType
  18 +from .util.Cache import Cache
  19 +
  20 +class Api(ApiTemplate):
  21 +
  22 + api_name = "refresh影像数据"
  23 +
  24 + def process(self):
  25 +
  26 + #可以注册一个目录
  27 + #返回结果
  28 + res = {}
  29 +
  30 + try:
  31 + guid = self.para.get("guid")
  32 + image:Image = Image.query.filter_by(guid=guid).one_or_none()
  33 +
  34 + image_service_info, zoo, servers = Cache.cache_data(None)
  35 + image_servers = image.server.split(",")
  36 + image_servers = [ser for ser in image_servers if ser in servers]
  37 +
  38 + data_server = image_servers[0]
  39 +
  40 +
  41 + if data_server.__eq__("本地服务器"):
  42 +
  43 + img: Dataset = gdal.Open(image.path, 0)
  44 + geo = img.GetGeoTransform()
  45 +
  46 + origin = osr.SpatialReference()
  47 + origin.ImportFromWkt(img.GetProjection())
  48 +
  49 + authority_code = origin.GetAuthorityCode(None)
  50 + band_count = img.RasterCount
  51 + band: Band = img.GetRasterBand(1)
  52 + count = band.GetOverviewCount()
  53 + nodatavalue = band.GetNoDataValue()
  54 + left_top = (geo[0], geo[3])
  55 +
  56 + right_buttom = (geo[0] + geo[1] * img.RasterXSize, geo[3] + geo[5] * img.RasterYSize)
  57 + origin_extent = [left_top[0], right_buttom[1], right_buttom[0], left_top[1]]
  58 +
  59 + update_info = {"band_count": band_count,
  60 + "band_view":image.band_view if band_count>=3 else "[1,1,1]",
  61 + "overview_count": count,
  62 + "raster_x_size": image.RasterXSize,
  63 + "raster_y_size": image.RasterYSize,
  64 + "cell_x_size": geo[1],
  65 + "cell_y_size": geo[5],
  66 + "extent": origin_extent,
  67 + "null_value": nodatavalue,
  68 + "size":os.path.getsize(image.path),
  69 + "crs_wkt": image.GetProjection(),
  70 + "crs": authority_code,
  71 + "crs_proj4": origin.ExportToProj4(),
  72 + "update_time":datetime.datetime.now()
  73 + }
  74 +
  75 + del img
  76 +
  77 + else:
  78 + thrift_connect = ThriftConnect(data_server)
  79 +
  80 + info = json.loads(thrift_connect.client.getInfo(image.path))
  81 + thrift_connect.close()
  82 +
  83 + update_info = {"band_count": info.get("band_count"),
  84 + "band_view":image.band_view if info.get("band_count")>=3 else "[1,1,1]",
  85 + "overview_count": info.get("overview_count"),
  86 + "raster_x_size" : info["xy_size"][0],
  87 + "raster_y_size" : info["xy_size"][1],
  88 + "cell_x_size" : info.get("cell_x_size"),
  89 + "cell_y_size" : abs(info.get("cell_y_size")),
  90 + "extent": json.dumps(info["origin_extent"]),
  91 + "null_value": info.get("null_value"),
  92 + "size":info.get("size"),
  93 + "crs" : str(info.get("crs")),
  94 + "crs_wkt" : info.get("crs_wkt"),
  95 + "crs_proj4" : info.get("crs_proj4"),
  96 + "update_time":datetime.datetime.now()
  97 + }
  98 + Image.query.filter_by(guid=guid).update(update_info)
  99 + db.session.commit()
  100 + except:
  101 + raise Exception("刷新影像数据信息失败！")
  102 +
  103 + res["result"] = True
  104 + return res
  105 +
  106 + api_doc = {
  107 + "tags": ["影像接口"],
  108 + "parameters": [
  109 + {"name": "guid",
  110 + "in": "formData",
  111 + "type": "string",
  112 + "description": "image_guid"},
  113 + ],
  114 + "responses": {
  115 + 200: {
  116 + "schema": {
  117 + "properties": {
  118 + }
  119 + }
  120 + }
  121 + }
  122 + }
  123 +
  124 +
@@ -39,6 +39,7 @@ class Api(ApiTemplate): @@ -39,6 +39,7 @@ class Api(ApiTemplate):
39 39
40 #注册某影像 40 #注册某影像
41 infos = [] 41 infos = []
  42 +
42 if data_server.__eq__("本地服务器"): 43 if data_server.__eq__("本地服务器"):
43 44
44 image_paths = list(self.recur_paths_local(paths)) 45 image_paths = list(self.recur_paths_local(paths))
@@ -62,14 +63,7 @@ class Api(ApiTemplate): @@ -62,14 +63,7 @@ class Api(ApiTemplate):
62 63
63 64
64 origin_extent = [left_top[0], right_buttom[1], right_buttom[0], left_top[1]] 65 origin_extent = [left_top[0], right_buttom[1], right_buttom[0], left_top[1]]
65 - # target = origin.CloneGeogCS()  
66 - # tran = osr.CoordinateTransformation(origin, target)  
67 - #  
68 - # geo_left_top = tran.TransformPoint(geo[0], geo[3])  
69 - # geo_right_buttom = tran.TransformPoint(geo[0] + geo[1] * image.RasterXSize,  
70 - # geo[3] + geo[5] * image.RasterYSize)  
71 - #  
72 - # geo_origin_extent = [geo_left_top[1], geo_right_buttom[0], geo_right_buttom[1], geo_left_top[0]] 66 +
73 67
74 info = {"band_count": band_count, 68 info = {"band_count": band_count,
75 "band_view":"[1,2,3]" if band_count>=3 else "[1,1,1]", 69 "band_view":"[1,2,3]" if band_count>=3 else "[1,1,1]",
@@ -77,12 +71,11 @@ class Api(ApiTemplate): @@ -77,12 +71,11 @@ class Api(ApiTemplate):
77 "path":image_info["path"], 71 "path":image_info["path"],
78 "xy_size": [image.RasterXSize, image.RasterYSize], 72 "xy_size": [image.RasterXSize, image.RasterYSize],
79 "origin_extent": origin_extent, 73 "origin_extent": origin_extent,
80 - # "geo_origin_extent": geo_origin_extent,  
81 "null_value": nodatavalue, 74 "null_value": nodatavalue,
82 "size":os.path.getsize(image_info["path"]), 75 "size":os.path.getsize(image_info["path"]),
83 - "sr_wkt": image.GetProjection(),  
84 - "epsg": authority_code,  
85 - "sr_proj4": origin.ExportToProj4(), 76 + "crs_wkt": image.GetProjection(),
  77 + "crs": authority_code,
  78 + "crs_proj4": origin.ExportToProj4(),
86 "cell_x_size": geo[1], 79 "cell_x_size": geo[1],
87 "cell_y_size": geo[5] 80 "cell_y_size": geo[5]
88 } 81 }
@@ -90,6 +83,7 @@ class Api(ApiTemplate): @@ -90,6 +83,7 @@ class Api(ApiTemplate):
90 infos.append(info) 83 infos.append(info)
91 del image 84 del image
92 85
  86 + #分布式下，从thrift获取信息
93 else: 87 else:
94 thrift_connect = ThriftConnect(data_server) 88 thrift_connect = ThriftConnect(data_server)
95 image_paths = list(self.recur_paths_remote(paths,thrift_connect.client)) 89 image_paths = list(self.recur_paths_remote(paths,thrift_connect.client))
@@ -98,22 +92,11 @@ class Api(ApiTemplate): @@ -98,22 +92,11 @@ class Api(ApiTemplate):
98 thrift_connect.close() 92 thrift_connect.close()
99 93
100 this_time = datetime.datetime.now() 94 this_time = datetime.datetime.now()
101 - for info in infos:  
102 95
103 - # 影像空间范围  
104 - # if not info["origin_extent"]:  
105 - # if not self.para.get("extent"):  
106 - # res["result"] = False  
107 - # res["msg"] = "数据解析范围失败，请手动输入范围"  
108 - # return res  
109 - # else :  
110 - # origin_extent=json.loads(self.para.get("extent"))  
111 - # else:  
112 - # origin_extent = info["origin_extent"]  
113 96
  97 + for info in infos:
114 98
115 - exist_image = Image.query.filter_by(path=os.path.normpath(info.get("path")),  
116 - size=info.get("size")).one_or_none() 99 + exist_image = Image.query.filter_by(path=os.path.normpath(info.get("path")), size=info.get("size")).one_or_none()
117 if exist_image: 100 if exist_image:
118 if exist_image.server.__contains__(data_server): 101 if exist_image.server.__contains__(data_server):
119 pass 102 pass
@@ -123,6 +106,7 @@ class Api(ApiTemplate): @@ -123,6 +106,7 @@ class Api(ApiTemplate):
123 else: 106 else:
124 img:Image = Image(guid= uuid.uuid1().__str__(), 107 img:Image = Image(guid= uuid.uuid1().__str__(),
125 overview_count=info.get("overview_count"), 108 overview_count=info.get("overview_count"),
  109 + has_pyramid = 1 if info.get("overview_count")>0 else 0,
126 raster_x_size=info["xy_size"][0], 110 raster_x_size=info["xy_size"][0],
127 raster_y_size=info["xy_size"][1], 111 raster_y_size=info["xy_size"][1],
128 cell_x_size = info.get("cell_x_size"), 112 cell_x_size = info.get("cell_x_size"),
@@ -57,12 +57,11 @@ class Api(ApiTemplate): @@ -57,12 +57,11 @@ class Api(ApiTemplate):
57 57
58 db.session.add(service) 58 db.session.add(service)
59 59
60 -  
61 image_service = ImageService(guid=image_service_guid, 60 image_service = ImageService(guid=image_service_guid,
62 name=name, 61 name=name,
63 create_time=this_time, 62 create_time=this_time,
64 scheme_guid=self.para.get("scheme_guid"), 63 scheme_guid=self.para.get("scheme_guid"),
65 - crs = self.para.get("crs"), 64 + crs = Image.query.filter_by(guid=guids[0]).one_or_none().crs,
66 service_guid=service_guid 65 service_guid=service_guid
67 ) 66 )
68 67
@@ -177,6 +176,14 @@ class Api(ApiTemplate): @@ -177,6 +176,14 @@ class Api(ApiTemplate):
177 "in": "formData", 176 "in": "formData",
178 "type": "string", 177 "type": "string",
179 "description": "[影像服务]功能类型，以逗号相隔，可选WMTS，WMS"}, 178 "description": "[影像服务]功能类型，以逗号相隔，可选WMTS，WMS"},
  179 + {"name": "cql",
  180 + "in": "formData",
  181 + "type": "string",
  182 + "description": "分层分级的CQL"},
  183 + {"name": "cql_table_guid",
  184 + "in": "formData",
  185 + "type": "string",
  186 + "description": "分层分级的CQL table_guid"},
180 187
181 ], 188 ],
182 "responses": { 189 "responses": {
@@ -39,6 +39,10 @@ class Api(ApiTemplate): @@ -39,6 +39,10 @@ class Api(ApiTemplate):
39 39
40 get_extent = parameter.get("get_extent") 40 get_extent = parameter.get("get_extent")
41 if get_extent and (get_extent == True or get_extent.lower().__eq__("true")): 41 if get_extent and (get_extent == True or get_extent.lower().__eq__("true")):
  42 +
  43 + sr_set = set()
  44 +
  45 +
42 tmp_extent = [] 46 tmp_extent = []
43 for g in image_guids.split(","): 47 for g in image_guids.split(","):
44 image = Image.query.filter_by(guid=g).one_or_none() 48 image = Image.query.filter_by(guid=g).one_or_none()
@@ -51,6 +55,11 @@ class Api(ApiTemplate): @@ -51,6 +55,11 @@ class Api(ApiTemplate):
51 tmp_extent[2] = max(image_extent[2], tmp_extent[2]) 55 tmp_extent[2] = max(image_extent[2], tmp_extent[2])
52 tmp_extent[1] = min(image_extent[1], tmp_extent[1]) 56 tmp_extent[1] = min(image_extent[1], tmp_extent[1])
53 tmp_extent[3] = max(image_extent[3], tmp_extent[3]) 57 tmp_extent[3] = max(image_extent[3], tmp_extent[3])
  58 + sr_set.add(image.crs)
  59 + if len(sr_set)>1:
  60 + result["result"] = False
  61 + result["msg"] = "影像坐标不一致"
  62 + return result
54 63
55 result["result"] = True 64 result["result"] = True
56 result["data"] = tmp_extent 65 result["data"] = tmp_extent
@@ -26,10 +26,25 @@ class Cache: @@ -26,10 +26,25 @@ class Cache:
26 zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=1) 26 zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=1)
27 zoo.start() 27 zoo.start()
28 GLOBAL_DIC["zookeeper"] = zoo 28 GLOBAL_DIC["zookeeper"] = zoo
  29 + GLOBAL_DIC["zookeeper_updatetime"] = time.time()
29 else: 30 else:
30 if not zoo.connected: 31 if not zoo.connected:
31 zoo.start() 32 zoo.start()
32 33
  34 + # 更新zoo
  35 + if not GLOBAL_DIC.get("zookeeper_updatetime"):
  36 + GLOBAL_DIC["zookeeper_updatetime"] = time.time()
  37 + if time.time() - GLOBAL_DIC["zookeeper_updatetime"] > 15:
  38 + #释放
  39 + try:
  40 + zoo.stop()
  41 + except:
  42 + pass
  43 + zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=1)
  44 + zoo.start()
  45 + GLOBAL_DIC["zookeeper"] = zoo
  46 + GLOBAL_DIC["zookeeper_updatetime"] = time.time()
  47 +
33 # 缓存数据服务器 48 # 缓存数据服务器
34 servers = GLOBAL_DIC.get("servers") 49 servers = GLOBAL_DIC.get("servers")
35 if servers is None: 50 if servers is None:
@@ -41,7 +56,7 @@ class Cache: @@ -41,7 +56,7 @@ class Cache:
41 servers = GLOBAL_DIC.get("servers") 56 servers = GLOBAL_DIC.get("servers")
42 57
43 # 更新缓存 58 # 更新缓存
44 - if time.time() - GLOBAL_DIC["servers_updatetime"] > 30: 59 + if time.time() - GLOBAL_DIC["servers_updatetime"] > 15:
45 servers = zoo.get_children("/rpc") 60 servers = zoo.get_children("/rpc")
46 servers.append("本地服务器") 61 servers.append("本地服务器")
47 GLOBAL_DIC["servers"] = servers 62 GLOBAL_DIC["servers"] = servers
@@ -51,7 +66,7 @@ class Cache: @@ -51,7 +66,7 @@ class Cache:
51 # 缓存服务信息 66 # 缓存服务信息
52 if guid_or_name: 67 if guid_or_name:
53 image_service_info = GLOBAL_DIC.get(guid_or_name) 68 image_service_info = GLOBAL_DIC.get(guid_or_name)
54 - if image_service_info is None or time.time() - GLOBAL_DIC.get("service_updatetime") > 30: 69 + if image_service_info is None or time.time() - GLOBAL_DIC.get("service_updatetime") > 15:
55 if type.__eq__("guid"): 70 if type.__eq__("guid"):
56 image_service: ImageService = ImageService.query.filter_by(guid=guid_or_name).one_or_none() 71 image_service: ImageService = ImageService.query.filter_by(guid=guid_or_name).one_or_none()
57 else: 72 else:
@@ -115,6 +115,8 @@ class Image(db.Model): @@ -115,6 +115,8 @@ class Image(db.Model):
115 cell_y_size = Column(Float)#像元y大小 115 cell_y_size = Column(Float)#像元y大小
116 region = Column(Text) 116 region = Column(Text)
117 117
  118 + has_pyramid = Column(Integer,default=0)#是否已经创建金字塔
  119 +
118 collect_time = Column(DateTime) #成像时间年份 120 collect_time = Column(DateTime) #成像时间年份
119 121
120 satellite = Column(String)#卫星类型 122 satellite = Column(String)#卫星类型
@@ -144,6 +146,10 @@ class ImageService(db.Model): @@ -144,6 +146,10 @@ class ImageService(db.Model):
144 create_time = Column(DateTime) 146 create_time = Column(DateTime)
145 service_guid = Column(String, ForeignKey('dmap_service.guid')) 147 service_guid = Column(String, ForeignKey('dmap_service.guid'))
146 148
  149 + #分层分级设置
  150 + # cql = Column(String(256))
  151 + # table_guid = Column(String(256))
  152 +
147 images = db.relationship('Image', 153 images = db.relationship('Image',
148 secondary=dmap_image_rel, 154 secondary=dmap_image_rel,
149 backref='image_services', 155 backref='image_services',
@@ -49,9 +49,6 @@ class Api(ApiTemplate): @@ -49,9 +49,6 @@ class Api(ApiTemplate):
49 if name: 49 if name:
50 services = services.filter(Service.name.like("%" + name + "%")) 50 services = services.filter(Service.name.like("%" + name + "%"))
51 51
52 - if onlyWMTS:  
53 - services = services.join(ServiceFunction).filter(ServiceFunction.type=="WMTS")  
54 -  
55 52
56 res["data"] = {} 53 res["data"] = {}
57 res["data"]["count"] = services.count() 54 res["data"]["count"] = services.count()
@@ -13,7 +13,6 @@ class StructurePrint: @@ -13,7 +13,6 @@ class StructurePrint:
13 singleton = None 13 singleton = None
14 f = None 14 f = None
15 15
16 -  
17 def __new__(cls, *args, **kwargs): 16 def __new__(cls, *args, **kwargs):
18 if not cls.singleton: 17 if not cls.singleton:
19 cls.singleton = super().__new__(cls) 18 cls.singleton = super().__new__(cls)
@@ -30,12 +29,6 @@ class StructurePrint: @@ -30,12 +29,6 @@ class StructurePrint:
30 self.f = open(self.log_file,"a",encoding="utf-8") 29 self.f = open(self.log_file,"a",encoding="utf-8")
31 30
32 31
33 - # def print(self,mes, type="info"):  
34 - # message = "[{}] {} {}\n".format(type.upper(), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), mes)  
35 - # self.f.write(message)  
36 - # dd =1  
37 -  
38 -  
39 def print(self,mes, type="info"): 32 def print(self,mes, type="info"):
40 with open(self.log_file,"a",encoding="utf-8") as f: 33 with open(self.log_file,"a",encoding="utf-8") as f:
41 message = "[{}] {} {}\n".format(type.upper(), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), mes) 34 message = "[{}] {} {}\n".format(type.upper(), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), mes)
@@ -5,7 +5,7 @@ @@ -5,7 +5,7 @@
5 5
6 6
7 import random 7 import random
8 -from .SliceScheme import SliceScheme 8 +from app.util.component.SliceScheme import SliceScheme
9 import random 9 import random
10 from osgeo.ogr import * 10 from osgeo.ogr import *
11 from osgeo import ogr 11 from osgeo import ogr
@@ -137,7 +137,28 @@ if __name__ == '__main__': @@ -137,7 +137,28 @@ if __name__ == '__main__':
137 # level = [8,15] 137 # level = [8,15]
138 # extent = [112, 22.7, 115, 24.03] 138 # extent = [112, 22.7, 115, 24.03]
139 139
140 - level = [12, 17]  
141 - extent = [111.644, 28.989, 111.7, 29.027]  
142 - output = "wmts.bat"  
143 - create(slice_para,extent,level,output)  
  140 + # level = [12, 17]
  141 + # extent = [111.644, 28.989, 111.7, 29.027]
  142 + # output = "wmts.bat"
  143 + # create(slice_para,extent,level,output)
  144 +
  145 +
  146 + slice_para = {'rows': 256.0, 'cols': 256.0, 'x': 10002100, 'y': -4923200, 'dpi': 96.0,
  147 + '9': {'scale': 1155583.4197265625, 'resolution': 305.7487246334356},
  148 + '10': {'scale': 577791.7098632812, 'resolution': 152.8743623167178},
  149 + '11': {'scale': 288895.8549316406, 'resolution': 76.4371811583589},
  150 + '12': {'scale': 144447.9274658203, 'resolution': 38.21859057917945},
  151 + '13': {'scale': 72223.96373291015, 'resolution': 19.109295289589724},
  152 + '14': {'scale': 36111.98186645508, 'resolution': 9.554647644794862},
  153 + '15': {'scale': 18055.99093322754, 'resolution': 4.777323822397431},
  154 + '16': {'scale': 9027.99546661377, 'resolution': 2.3886619111987155},
  155 + '17': {'scale': 4513.997733306885, 'resolution': 1.1943309555993578},
  156 + '18': {'scale': 2256.9988666534423, 'resolution': 0.5971654777996789}}
  157 +
  158 + # level = [9, 18]
  159 + # extent = [12366899.680315087, 2387281.267402596, 12870772.570770962, 2807990.6710842005]
  160 + # output = "ghcwmts.bat"
  161 + #
  162 + # create(slice_para,extent,level,output)
  163 +
  164 + print(SliceScheme.get_polygon(slice_para,13,1488,3564))
@@ -4,7 +4,8 @@ import logging @@ -4,7 +4,8 @@ import logging
4 deploy_ip_host = "172.26.40.105:8840" 4 deploy_ip_host = "172.26.40.105:8840"
5 # 系统数据库 5 # 系统数据库
6 6
7 -SQLALCHEMY_DATABASE_URI = "postgresql://postgres:chinadci@172.26.99.160:5432/dmap_dms_test" 7 +# SQLALCHEMY_DATABASE_URI = "postgresql://postgres:chinadci@172.26.60.100:5432/dmap_manager_test"
  8 +SQLALCHEMY_DATABASE_URI = "postgresql://postgres:chinadci@172.26.60.100:5432/dmap_manager"
8 # SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5433/dmap_dms_test" 9 # SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5433/dmap_dms_test"
9 10
10 # 指定精华表所在位置(必须为空间库)，设置为None则存放在各自的实体库中 11 # 指定精华表所在位置(必须为空间库)，设置为None则存放在各自的实体库中
@@ -18,3 +18,4 @@ thrift==0.13.0 @@ -18,3 +18,4 @@ thrift==0.13.0
18 Authlib==0.13 18 Authlib==0.13
19 kazoo==2.8.0 19 kazoo==2.8.0
20 paramiko==2.8.0 20 paramiko==2.8.0
  21 +requests==2.26.0
请 注册 或 登录 后发表评论