HWJ 何伟俊 / DMapManager · 提交

提交 6f523f46463309e42a149216d39034e7d9418384

作者 nheweijun
1 个父辈 adc1dc44

优化抽稀功能，超大的图形可以额外加入到抽稀图层中

内嵌 并排对比
正在显示 10 个修改的文件 包含 127 行增加 和 97 行删除
@@ -107,44 +107,48 @@ class Api(ApiTemplate): @@ -107,44 +107,48 @@ class Api(ApiTemplate):
107 for i in range(pg_ds.GetLayerCount()): 107 for i in range(pg_ds.GetLayerCount()):
108 layer: Layer = pg_ds.GetLayer(i) 108 layer: Layer = pg_ds.GetLayer(i)
109 l_name = layer.GetName() 109 l_name = layer.GetName()
  110 + try:
110 111
111 - # 只注册public的空间表,其他表空间的表名会有.  
112 - if layer.GetName().__contains__("."):  
113 - continue  
114 -  
115 - # 不注册抽稀表  
116 - if layer.GetName().__contains__("_vacuate_"):  
117 - spatial_table_name.append(layer.GetName())  
118 - continue 112 + # 只注册public的空间表,其他表空间的表名会有.
  113 + if layer.GetName().__contains__("."):
  114 + continue
119 115
120 - # 没有权限的表跳过  
121 - if not PGUtil.check_table_privilege(l_name,"SELECT",db_tuple[0],pg_ds):  
122 - StructurePrint.print("用户{}对表{}没有select权限！".format(db_tuple[0],l_name),"warn")  
123 - continue 116 + # 不注册抽稀表
  117 + if layer.GetName().__contains__("_vacuate_"):
  118 + spatial_table_name.append(layer.GetName())
  119 + continue
124 120
125 - # 范围统计和数量统计以100w为界限  
126 - query_count_layer: Layer = pg_ds.ExecuteSQL(  
127 - '''SELECT reltuples::bigint AS ec FROM pg_class WHERE oid = 'public."{}"'::regclass'''.format(l_name)) 121 + # 没有权限的表跳过
  122 + if not PGUtil.check_table_privilege(l_name,"SELECT",db_tuple[0],pg_ds):
  123 + StructurePrint.print("用户{}对表{}没有select权限！".format(db_tuple[0],l_name),"warn")
  124 + continue
128 125
129 - feature_count = query_count_layer.GetFeature(0).GetField("ec")  
130 - # 要素少于100w可以精确统计  
131 - if feature_count < 1000000:  
132 - feature_count = layer.GetFeatureCount()  
133 - ext = layer.GetExtent()  
134 - else:  
135 - query_ext_layer: Layer = pg_ds.ExecuteSQL(  
136 - "select geometry(ST_EstimatedExtent('public', '{}','{}'))".format(l_name,  
137 - layer.GetGeometryColumn()))  
138 - ext = query_ext_layer.GetExtent()  
139 - if ext[0] < 360:  
140 - ext = [round(e, 6) for e in ext]  
141 - else:  
142 - ext = [round(e, 2) for e in ext]  
143 - extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) 126 + # 范围统计和数量统计以100w为界限
  127 + query_count_layer: Layer = pg_ds.ExecuteSQL(
  128 + '''SELECT reltuples::bigint AS ec FROM pg_class WHERE oid = 'public."{}"'::regclass'''.format(l_name))
  129 +
  130 + feature_count = query_count_layer.GetFeature(0).GetField("ec")
  131 + # 要素少于100w可以精确统计
  132 + if feature_count < 1000000:
  133 + feature_count = layer.GetFeatureCount()
  134 + ext = layer.GetExtent()
  135 + else:
  136 + query_ext_layer: Layer = pg_ds.ExecuteSQL(
  137 + "select geometry(ST_EstimatedExtent('public', '{}','{}'))".format(l_name,
  138 + layer.GetGeometryColumn()))
  139 + ext = query_ext_layer.GetExtent()
  140 + if ext[0] < 360:
  141 + ext = [round(e, 6) for e in ext]
  142 + else:
  143 + ext = [round(e, 2) for e in ext]
  144 + extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3])
144 145
145 - table_guid = uuid.uuid1().__str__() 146 + table_guid = uuid.uuid1().__str__()
146 147
147 - geom_type = GeometryAdapter.get_geometry_type(layer) 148 + geom_type = GeometryAdapter.get_geometry_type(layer)
  149 + except:
  150 + StructurePrint.print("表{}注册失败！".format(l_name), "warn")
  151 + continue
148 152
149 table = Table(guid=table_guid, 153 table = Table(guid=table_guid,
150 database_guid=database.guid, 154 database_guid=database.guid,
@@ -14,12 +14,15 @@ from app.models import db @@ -14,12 +14,15 @@ from app.models import db
14 from app.util.component.ApiTemplate import ApiTemplate 14 from app.util.component.ApiTemplate import ApiTemplate
15 15
16 from app.util.component.PGUtil import PGUtil 16 from app.util.component.PGUtil import PGUtil
  17 +import configure
  18 +from flask import current_app
17 class Api(ApiTemplate): 19 class Api(ApiTemplate):
18 api_name = "删除表" 20 api_name = "删除表"
19 def process(self): 21 def process(self):
20 22
21 res = {} 23 res = {}
22 pg_ds = None 24 pg_ds = None
  25 + sys_ds = None
23 try: 26 try:
24 27
25 table_guid = self.para.get("guid") 28 table_guid = self.para.get("guid")
@@ -45,14 +48,22 @@ class Api(ApiTemplate): @@ -45,14 +48,22 @@ class Api(ApiTemplate):
45 48
46 49
47 pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri)) 50 pg_ds: DataSource = PGUtil.open_pg_data_source(1, DES.decode(database.sqlalchemy_uri))
48 - 51 + sys_ds: DataSource = PGUtil.open_pg_data_source(1, configure.SQLALCHEMY_DATABASE_URI)
49 52
50 #删除抽稀表 53 #删除抽稀表
51 vacuate_tables=table.relate_table_vacuates.all() 54 vacuate_tables=table.relate_table_vacuates.all()
52 for vt in vacuate_tables: 55 for vt in vacuate_tables:
53 - pg_ds.DeleteLayer(vt.name) 56 + db.session.delete(vt)
  57 + try:
  58 + sys_ds.DeleteLayer(vt.name)
  59 + except:
  60 + current_app.logger.warning("{}不存在！".format(vt.name))
  61 +
  62 + try:
  63 + pg_ds.DeleteLayer(table.name)
  64 + except:
  65 + current_app.logger.warning("{}不存在！".format(table.name))
54 66
55 - pg_ds.DeleteLayer(table.name)  
56 67
57 # 删除元数据 68 # 删除元数据
58 69
@@ -65,6 +76,8 @@ class Api(ApiTemplate): @@ -65,6 +76,8 @@ class Api(ApiTemplate):
65 finally: 76 finally:
66 if pg_ds: 77 if pg_ds:
67 pg_ds.Destroy() 78 pg_ds.Destroy()
  79 + if sys_ds:
  80 + sys_ds.Destroy()
68 return res 81 return res
69 82
70 api_doc={ 83 api_doc={
@@ -13,10 +13,10 @@ class Api(ApiTemplate): @@ -13,10 +13,10 @@ class Api(ApiTemplate):
13 res = {} 13 res = {}
14 try: 14 try:
15 table_guid = self.para.get("guid") 15 table_guid = self.para.get("guid")
16 - table = Table.query.filter_by(guid=table_guid)  
17 - if not table.one_or_none(): 16 + table = Table.query.filter_by(guid=table_guid).one_or_none()
  17 + if not table:
18 raise Exception("数据不存在！") 18 raise Exception("数据不存在！")
19 - table = table.one_or_none() 19 +
20 columns = table.relate_columns.order_by(Columns.name).all() 20 columns = table.relate_columns.order_by(Columns.name).all()
21 res["data"]=ModelVisitor.table_to_json(table) 21 res["data"]=ModelVisitor.table_to_json(table)
22 res["data"]["columns"] = ModelVisitor.objects_to_jsonarray(columns) 22 res["data"]["columns"] = ModelVisitor.objects_to_jsonarray(columns)
@@ -189,43 +189,47 @@ class Api(ApiTemplate): @@ -189,43 +189,47 @@ class Api(ApiTemplate):
189 if layer.GetName() not in spatial_tables_names: 189 if layer.GetName() not in spatial_tables_names:
190 l_name = layer.GetName() 190 l_name = layer.GetName()
191 191
192 - # 只注册public的空间表,其他表空间的表名会有.  
193 - if layer.GetName().__contains__("."):  
194 - continue  
195 - # 略过抽稀表  
196 - if layer.GetName().__contains__("_vacuate_"):  
197 - continue  
198 -  
199 - # 没有权限的表跳过  
200 - if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds):  
201 - StructurePrint.print("用户{}对表{}没有select权限！".format(db_tuple[0], l_name), "warn")  
202 - continue  
203 -  
204 -  
205 - # 范围统计和数量统计以100w为界限  
206 - query_count_layer: Layer = pg_ds.ExecuteSQL(  
207 - '''SELECT reltuples::bigint AS ec FROM pg_class WHERE oid = 'public."{}"'::regclass'''.format(  
208 - l_name)) 192 + try:
  193 + # 只注册public的空间表,其他表空间的表名会有.
  194 + if layer.GetName().__contains__("."):
  195 + continue
  196 + # 略过抽稀表
  197 + if layer.GetName().__contains__("_vacuate_"):
  198 + continue
209 199
210 - feature_count = query_count_layer.GetFeature(0).GetField("ec")  
211 - # 要素少于100w可以精确统计  
212 - if feature_count < 1000000:  
213 - feature_count = layer.GetFeatureCount()  
214 - ext = layer.GetExtent()  
215 - else:  
216 - query_ext_layer: Layer = pg_ds.ExecuteSQL(  
217 - "select geometry(ST_EstimatedExtent('public', '{}','{}'))".format(l_name,  
218 - layer.GetGeometryColumn()))  
219 - ext = query_ext_layer.GetExtent()  
220 - if ext[0] < 360:  
221 - ext = [round(e, 6) for e in ext]  
222 - else:  
223 - ext = [round(e, 2) for e in ext]  
224 - extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) 200 + # 没有权限的表跳过
  201 + if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds):
  202 + StructurePrint.print("用户{}对表{}没有select权限！".format(db_tuple[0], l_name), "warn")
  203 + continue
225 204
226 - StructurePrint.print("空间表增加！")  
227 205
228 - geom_type = GeometryAdapter.get_geometry_type(layer) 206 + # 范围统计和数量统计以100w为界限
  207 + query_count_layer: Layer = pg_ds.ExecuteSQL(
  208 + '''SELECT reltuples::bigint AS ec FROM pg_class WHERE oid = 'public."{}"'::regclass'''.format(
  209 + l_name))
  210 +
  211 + feature_count = query_count_layer.GetFeature(0).GetField("ec")
  212 + # 要素少于100w可以精确统计
  213 + if feature_count < 1000000:
  214 + feature_count = layer.GetFeatureCount()
  215 + ext = layer.GetExtent()
  216 + else:
  217 + query_ext_layer: Layer = pg_ds.ExecuteSQL(
  218 + "select geometry(ST_EstimatedExtent('public', '{}','{}'))".format(l_name,
  219 + layer.GetGeometryColumn()))
  220 + ext = query_ext_layer.GetExtent()
  221 + if ext[0] < 360:
  222 + ext = [round(e, 6) for e in ext]
  223 + else:
  224 + ext = [round(e, 2) for e in ext]
  225 + extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3])
  226 +
  227 + StructurePrint.print("空间表增加！")
  228 +
  229 + geom_type = GeometryAdapter.get_geometry_type(layer)
  230 + except:
  231 + StructurePrint.print("表{}注册失败！".format(l_name), "warn")
  232 + continue
229 233
230 table_guid = uuid.uuid1().__str__() 234 table_guid = uuid.uuid1().__str__()
231 table = Table(guid=table_guid, 235 table = Table(guid=table_guid,
@@ -293,8 +293,8 @@ class VacuateProcess: @@ -293,8 +293,8 @@ class VacuateProcess:
293 293
294 v_ln = "z{}_vacuate_{}_{}".format(table_guid,lev, grid_name) 294 v_ln = "z{}_vacuate_{}_{}".format(table_guid,lev, grid_name)
295 vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) 295 vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options)
296 - # 抽稀表不需要属性  
297 - # vl.CreateFields(schema) 296 + # 抽稀表需要属性
  297 + vl.CreateFields(layer.schema)
298 self.vacuate_layers[l] = vl 298 self.vacuate_layers[l] = vl
299 299
300 else: 300 else:
@@ -165,19 +165,20 @@ def clip(): @@ -165,19 +165,20 @@ def clip():
165 raise Exception("打开数据失败！") 165 raise Exception("打开数据失败！")
166 166
167 output_ds: DataSource = shp_driver.CreateDataSource(output) 167 output_ds: DataSource = shp_driver.CreateDataSource(output)
168 -  
169 - out_layer: Layer = output_ds.CreateLayer(layer.GetName(), layer.GetSpatialRef(), layer.GetGeomType())  
170 - out_layer.CreateFields(layer.schema) 168 + #
  169 + # out_layer: Layer = output_ds.CreateLayer(layer.GetName(), layer.GetSpatialRef(), layer.GetGeomType())
  170 + # out_layer.CreateFields(layer.schema)
171 for f in layer: 171 for f in layer:
172 - new_f = copy.copy(f) 172 + new_f:Feature = copy.copy(f)
173 out_g = f.GetGeometryRef() 173 out_g = f.GetGeometryRef()
174 if out_g.Intersect(bound_geo): 174 if out_g.Intersect(bound_geo):
175 out_g = out_g.Intersection(bound_geo) 175 out_g = out_g.Intersection(bound_geo)
176 new_f.SetGeometry(out_g) 176 new_f.SetGeometry(out_g)
  177 + new_f.SetField("ttt",1)
177 # new_f.UnsetField("OBJECTID") 178 # new_f.UnsetField("OBJECTID")
178 - out_layer.CreateFeature(new_f) 179 + # out_layer.CreateFeature(new_f)
179 180
180 - output_ds.Destroy() 181 + # output_ds.Destroy()
181 182
182 183
183 184
@@ -186,7 +187,7 @@ if __name__ == '__main__': @@ -186,7 +187,7 @@ if __name__ == '__main__':
186 # input = r"E:\Data\copy\origin.shp" 187 # input = r"E:\Data\copy\origin.shp"
187 # output = r"E:\Data\copy\re.shp" 188 # output = r"E:\Data\copy\re.shp"
188 # copydata(bound_shp,input,output) 189 # copydata(bound_shp,input,output)
189 - 190 + #
190 # bound_shp = "/root/CopyData/gdsample.shp" 191 # bound_shp = "/root/CopyData/gdsample.shp"
191 # input = r"/root/CopyData/origin.shp" 192 # input = r"/root/CopyData/origin.shp"
192 # output = r"/root/CopyData/re.gdb" 193 # output = r"/root/CopyData/re.gdb"
@@ -157,7 +157,7 @@ class EntryDataVacuate: @@ -157,7 +157,7 @@ class EntryDataVacuate:
157 157
158 # this_task.pg_ds.StartTransaction() 158 # this_task.pg_ds.StartTransaction()
159 new_layer_name = None 159 new_layer_name = None
160 - vacuate_process= None 160 + # vacuate_process= None
161 success = True 161 success = True
162 table_guid = uuid.uuid1().__str__() 162 table_guid = uuid.uuid1().__str__()
163 try: 163 try:
@@ -200,7 +200,7 @@ class EntryDataVacuate: @@ -200,7 +200,7 @@ class EntryDataVacuate:
200 pg_layer.CreateFields(schema) 200 pg_layer.CreateFields(schema)
201 201
202 #创建抽稀过程 202 #创建抽稀过程
203 - vacuate_process = VacuateProcess(layer,table_guid,options) 203 + # vacuate_process = VacuateProcess(layer,table_guid,options)
204 204
205 205
206 206
@@ -229,17 +229,18 @@ class EntryDataVacuate: @@ -229,17 +229,18 @@ class EntryDataVacuate:
229 pg_layer.CreateFeature(out_feature) 229 pg_layer.CreateFeature(out_feature)
230 230
231 #插入抽稀图层 231 #插入抽稀图层
232 - if out_geom is not None:  
233 - vacuate_process.vacuate(out_geom) 232 + # if out_geom is not None:
  233 + # vacuate_process.vacuate(out_geom)
234 234
235 # 注册图层信息 235 # 注册图层信息
236 # 是否抽吸过 236 # 是否抽吸过
237 - is_vacuate = 1 if vacuate_process.max_level>0 else 0 237 + # is_vacuate = 1 if vacuate_process.max_level>0 else 0
  238 + is_vacuate = 0
238 239
239 this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) 240 this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid)
240 241
241 # 注册抽稀表 242 # 注册抽稀表
242 - this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) 243 + # this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers)
243 244
244 this_task.write_process("{}图层入库成功。".format(new_layer_name)) 245 this_task.write_process("{}图层入库成功。".format(new_layer_name))
245 246
@@ -249,11 +250,12 @@ class EntryDataVacuate: @@ -249,11 +250,12 @@ class EntryDataVacuate:
249 StructurePrint.print("{}入库失败，数据回滚！原因：{}".format(new_layer_name,e.__str__()), "error") 250 StructurePrint.print("{}入库失败，数据回滚！原因：{}".format(new_layer_name,e.__str__()), "error")
250 print(traceback.format_exc()) 251 print(traceback.format_exc())
251 # 抽稀回滚 252 # 抽稀回滚
252 - vacuate_process.rollback() 253 + # vacuate_process.rollback()
253 success =False 254 success =False
254 255
255 finally: 256 finally:
256 - vacuate_process.end() 257 + # vacuate_process.end()
  258 + pass
257 return success,new_layer_name 259 return success,new_layer_name
258 260
259 261
@@ -516,8 +518,8 @@ class VacuateProcess: @@ -516,8 +518,8 @@ class VacuateProcess:
516 518
517 v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name) 519 v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name)
518 vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) 520 vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options)
519 - # 抽稀表不需要属性  
520 - # vl.CreateFields(schema) 521 + # 抽稀表需要属性
  522 + vl.CreateFields(layer.schema)
521 self.vacuate_layers[l] = vl 523 self.vacuate_layers[l] = vl
522 524
523 else: 525 else:
@@ -3,11 +3,17 @@ @@ -3,11 +3,17 @@
3 #createtime: 2021/5/17 3 #createtime: 2021/5/17
4 #email: nheweijun@sina.com 4 #email: nheweijun@sina.com
5 import datetime 5 import datetime
6 - 6 +import os
7 7
8 class StructurePrint: 8 class StructurePrint:
9 9
  10 + log_file = os.path.join(os.path.dirname(
  11 + os.path.dirname(
  12 + os.path.dirname(
  13 + os.path.dirname(
  14 + os.path.realpath(__file__))))), "logs", "log.txt")
10 @classmethod 15 @classmethod
11 def print(cls,mes, type="info"): 16 def print(cls,mes, type="info"):
12 - message = "[{}] {} {}".format(type.upper(), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), mes)  
13 - print(message) 17 + with open(cls.log_file,"a",encoding="utf-8") as f:
  18 + message = "[{}] {} {}\n".format(type.upper(), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), mes)
  19 + f.write(message)
@@ -4,9 +4,11 @@ @@ -4,9 +4,11 @@
4 deploy_ip_host = "172.26.99.160:8840" 4 deploy_ip_host = "172.26.99.160:8840"
5 # 系统数据库 5 # 系统数据库
6 6
7 -# SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@172.26.40.254:5433/dmap_dms_test"  
8 SQLALCHEMY_DATABASE_URI = "postgresql://postgres:chinadci@172.26.99.160:5432/dmap_dms_test" 7 SQLALCHEMY_DATABASE_URI = "postgresql://postgres:chinadci@172.26.99.160:5432/dmap_dms_test"
9 8
  9 +# 指定精华表所在位置(必须为空间库)，设置为None则存放在各自的实体库中
  10 +VACUATE_DB_URI = None
  11 +# VACUATE_DB_URI = SQLALCHEMY_DATABASE_URI
10 12
11 # 部署模式cluster，standalone 13 # 部署模式cluster，standalone
12 deployment_mode = "cluster" 14 deployment_mode = "cluster"
请 注册 或 登录 后发表评论