正在显示
14 个修改的文件
包含
341 行增加
和
80 行删除
... | ... | @@ -4,29 +4,51 @@ |
4 | 4 | #email: nheweijun@sina.com |
5 | 5 | |
6 | 6 | |
7 | -from app.models import Database,db | |
7 | +from app.models import Database,db,Table,TableVacuate | |
8 | 8 | |
9 | 9 | |
10 | 10 | from app.util.component.ApiTemplate import ApiTemplate |
11 | +from app.util.component.PGUtil import PGUtil | |
12 | +import configure | |
13 | +from osgeo.ogr import DataSource | |
14 | +from flask import current_app | |
11 | 15 | class Api(ApiTemplate): |
12 | 16 | api_name = "删除数据库" |
13 | 17 | def process(self): |
14 | - res ={} | |
18 | + re ={} | |
19 | + sys_ds = None | |
15 | 20 | try: |
16 | 21 | |
17 | 22 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() |
23 | + sys_ds :DataSource= PGUtil.open_pg_data_source(1,configure.SQLALCHEMY_DATABASE_URI) | |
18 | 24 | if database: |
25 | + # 删除table | |
26 | + re_tables = Table.query.filter_by(database_guid=database.guid).all() | |
27 | + for table in re_tables: | |
28 | + re_tvs = TableVacuate.query.filter_by(table_guid=table.guid).all() | |
29 | + for tv in re_tvs: | |
30 | + db.session.delete(tv) | |
31 | + try: | |
32 | + sys_ds.DeleteLayer(tv.name) | |
33 | + except: | |
34 | + current_app.logger.warning("{}不存在".format(tv.name)) | |
35 | + db.session.delete(table) | |
19 | 36 | db.session.delete(database) |
37 | + | |
20 | 38 | db.session.commit() |
21 | - res["msg"] = "数据库删除成功!" | |
22 | - res["result"] = True | |
39 | + re["result"] = True | |
40 | + re["msg"] = "数据库删除成功!" | |
23 | 41 | else: |
24 | - res["msg"] = "数据库不存在!" | |
25 | - res["result"] = False | |
42 | + re["result"] = False | |
43 | + re["msg"] = "数据库不存在!" | |
26 | 44 | except Exception as e: |
27 | 45 | db.session.rollback() |
46 | + sys_ds.RollbackTransaction() | |
28 | 47 | raise e |
29 | - return res | |
48 | + finally: | |
49 | + if sys_ds: | |
50 | + sys_ds.Destroy() | |
51 | + return re | |
30 | 52 | |
31 | 53 | |
32 | 54 | ... | ... |
... | ... | @@ -17,6 +17,7 @@ from app.util.component.PGUtil import PGUtil |
17 | 17 | from app.util.component.SQLUtil import SQLUtil |
18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
19 | 19 | from app.util.component.StructuredPrint import StructurePrint |
20 | +import configure | |
20 | 21 | class Api(ApiTemplate): |
21 | 22 | api_name = "注册数据库" |
22 | 23 | def process(self): |
... | ... | @@ -57,7 +58,6 @@ class Api(ApiTemplate): |
57 | 58 | elif not self.check_space(sqlalchemy_uri): |
58 | 59 | res["msg"] = "数据不是空间数据库!" |
59 | 60 | return res |
60 | - | |
61 | 61 | else: |
62 | 62 | this_time = datetime.datetime.now() |
63 | 63 | database_guid = uuid.uuid1().__str__() |
... | ... | @@ -96,12 +96,8 @@ class Api(ApiTemplate): |
96 | 96 | # 注册空间表 |
97 | 97 | spatial_table_name,tables = self.register_spatial_table(pg_ds, database, this_time,db_tuple) |
98 | 98 | |
99 | - #注册抽稀表 | |
100 | - self.regiser_vacuate_table(pg_ds,tables,db_tuple) | |
101 | - | |
102 | 99 | #注册普通表 |
103 | 100 | self.register_common_table(this_time,database,spatial_table_name,db_tuple) |
104 | - | |
105 | 101 | pg_ds.Destroy() |
106 | 102 | |
107 | 103 | |
... | ... | @@ -245,11 +241,11 @@ class Api(ApiTemplate): |
245 | 241 | if l_name.__contains__("_vacuate_"): |
246 | 242 | |
247 | 243 | # 没有权限的表跳过 |
248 | - if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
249 | - StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
250 | - continue | |
244 | + # if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
245 | + # StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
246 | + # continue | |
251 | 247 | |
252 | - base_layer_name=l_name.split("_vacuate_")[0] | |
248 | + base_layer_name=l_name.split("_vacuate_")[1] | |
253 | 249 | level = l_name.split("_")[-2] |
254 | 250 | pixel_distance_str: str ="0" |
255 | 251 | try: | ... | ... |
... | ... | @@ -15,7 +15,6 @@ import uuid |
15 | 15 | from sqlalchemy.orm import Session |
16 | 16 | from app.util.component.SQLUtil import SQLUtil |
17 | 17 | from app.util.component.PGUtil import PGUtil |
18 | -from app.util.component.ModelVisitor import ModelVisitor | |
19 | 18 | from app.util.component.StructuredPrint import StructurePrint |
20 | 19 | from app.util.component.ApiTemplate import ApiTemplate |
21 | 20 | from app.util.component.GeometryAdapter import GeometryAdapter |
... | ... | @@ -37,8 +36,6 @@ class Api(ApiTemplate): |
37 | 36 | # 初始化task |
38 | 37 | task_guid = uuid.uuid1().__str__() |
39 | 38 | |
40 | - # result = self.table_refresh(database) | |
41 | - | |
42 | 39 | refresh_process = multiprocessing.Process(target=self.table_refresh,args=(database,task_guid)) |
43 | 40 | refresh_process.start() |
44 | 41 | |
... | ... | @@ -67,6 +64,7 @@ class Api(ApiTemplate): |
67 | 64 | def table_refresh(self,database,task_guid): |
68 | 65 | |
69 | 66 | pg_ds =None |
67 | + sys_ds =None | |
70 | 68 | data_session=None |
71 | 69 | result = {} |
72 | 70 | sys_session = None |
... | ... | @@ -74,7 +72,7 @@ class Api(ApiTemplate): |
74 | 72 | |
75 | 73 | try: |
76 | 74 | sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) |
77 | - | |
75 | + sys_ds = PGUtil.open_pg_data_source(0,configure.SQLALCHEMY_DATABASE_URI) | |
78 | 76 | |
79 | 77 | this_time = datetime.datetime.now() |
80 | 78 | database_guid = database.guid |
... | ... | @@ -99,8 +97,6 @@ class Api(ApiTemplate): |
99 | 97 | # 空间表处理完毕 |
100 | 98 | sys_session.commit() |
101 | 99 | |
102 | - # 处理抽稀表 | |
103 | - self.deal_vacuate_table(pg_ds,sys_session, database.guid,db_tuple) | |
104 | 100 | |
105 | 101 | # 空间表处理完毕 |
106 | 102 | sys_session.commit() |
... | ... | @@ -168,6 +164,8 @@ class Api(ApiTemplate): |
168 | 164 | data_session.close() |
169 | 165 | if sys_session: |
170 | 166 | sys_session.close() |
167 | + if sys_ds: | |
168 | + sys_ds.Destroy() | |
171 | 169 | return result |
172 | 170 | |
173 | 171 | def add_spatail_table(self,database,pg_ds,sys_session,spatial_tables_names,this_time,db_tuple): |
... | ... | @@ -251,11 +249,11 @@ class Api(ApiTemplate): |
251 | 249 | sys_session.add(column) |
252 | 250 | return db_tables_names |
253 | 251 | |
254 | - def deal_vacuate_table(self,pg_ds,sys_session,database_guid,db_tuple): | |
252 | + def deal_vacuate_table(self,sys_ds,sys_session,database_guid): | |
255 | 253 | |
256 | 254 | |
257 | - for i in range(pg_ds.GetLayerCount()): | |
258 | - layer: Layer = pg_ds.GetLayer(i) | |
255 | + for i in range(sys_ds.GetLayerCount()): | |
256 | + layer: Layer = sys_ds.GetLayer(i) | |
259 | 257 | geom_column = layer.GetGeometryColumn() |
260 | 258 | |
261 | 259 | if not geom_column: |
... | ... | @@ -266,12 +264,8 @@ class Api(ApiTemplate): |
266 | 264 | if layer.GetName().__contains__("_vacuate_"): |
267 | 265 | l_name = layer.GetName() |
268 | 266 | |
269 | - # 没有权限的表跳过 | |
270 | - if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
271 | - StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
272 | - continue | |
267 | + base_layer_name = l_name.split("_vacuate_")[0].split("_")[1] | |
273 | 268 | |
274 | - base_layer_name = l_name.split("_vacuate_")[0] | |
275 | 269 | level = l_name.split("_")[-2] |
276 | 270 | |
277 | 271 | pixel_distance_str: str ="0" | ... | ... |
... | ... | @@ -132,7 +132,7 @@ class Api(ApiTemplate): |
132 | 132 | |
133 | 133 | layer = pg_ds.GetLayerByName(table.name) |
134 | 134 | |
135 | - vacuate_process:VacuateProcess = VacuateProcess(layer, table.name, options,database.sqlalchemy_uri) | |
135 | + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options) | |
136 | 136 | |
137 | 137 | |
138 | 138 | for feature in layer: | ... | ... |
... | ... | @@ -18,6 +18,7 @@ import uuid |
18 | 18 | import configure |
19 | 19 | from osgeo.ogr import DataSource,Layer,Geometry |
20 | 20 | from osgeo import ogr |
21 | +from flask import current_app | |
21 | 22 | |
22 | 23 | class Api(ApiTemplate): |
23 | 24 | api_name = "删除抽稀表" |
... | ... | @@ -26,6 +27,7 @@ class Api(ApiTemplate): |
26 | 27 | res = {} |
27 | 28 | res["data"] = {} |
28 | 29 | pg_ds = None |
30 | + sys_ds = None | |
29 | 31 | try: |
30 | 32 | table_guid = self.para.get("guid") |
31 | 33 | table: Table = Table.query.filter_by(guid=table_guid).one_or_none() |
... | ... | @@ -40,13 +42,9 @@ class Api(ApiTemplate): |
40 | 42 | |
41 | 43 | # 判断图层是否存在 |
42 | 44 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
43 | - layer = pg_ds.GetLayerByName(table.name) | |
45 | + sys_ds :DataSource= PGUtil.open_pg_data_source(0,configure.SQLALCHEMY_DATABASE_URI) | |
44 | 46 | |
45 | - # 判断用户权限 | |
46 | - # user, pw, host, port, database = PGUtil.get_info_from_sqlachemy_uri(DES.decode(table.relate_database.sqlalchemy_uri)) | |
47 | - # query_role:Layer = pg_ds.ExecuteSQL("SELECT rolcreatedb FROM pg_roles WHERE rolname='{}'".format(user)) | |
48 | - # if not query_role.GetFeature(0).GetField("rolcreatedb"): | |
49 | - # raise Exception("用户不具备建表权限!") | |
47 | + layer = pg_ds.GetLayerByName(table.name) | |
50 | 48 | |
51 | 49 | if not layer: |
52 | 50 | raise Exception("图层不存在!") |
... | ... | @@ -58,9 +56,9 @@ class Api(ApiTemplate): |
58 | 56 | for tv in tvs : |
59 | 57 | db.session.delete(tv) |
60 | 58 | try: |
61 | - pg_ds.DeleteLayer(tv.name) | |
59 | + sys_ds.DeleteLayer(tv.name) | |
62 | 60 | except Exception as e : |
63 | - StructurePrint.print("抽稀图层不存在!","warn") | |
61 | + current_app.logger.warning("抽稀图层不存在!") | |
64 | 62 | db.session.commit() |
65 | 63 | res["result"] = True |
66 | 64 | |
... | ... | @@ -69,6 +67,8 @@ class Api(ApiTemplate): |
69 | 67 | finally: |
70 | 68 | if pg_ds: |
71 | 69 | pg_ds.Destroy() |
70 | + if sys_ds: | |
71 | + sys_ds.Destroy() | |
72 | 72 | return res |
73 | 73 | |
74 | 74 | ... | ... |
... | ... | @@ -25,8 +25,10 @@ class Api(ApiTemplate): |
25 | 25 | if not table: |
26 | 26 | raise Exception("数据不存在!") |
27 | 27 | tvs = TableVacuate.query.filter_by(table_guid=table_guid).all() |
28 | - dat = collections.OrderedDict() | |
29 | - if float(table.extent.split(",")[0])<180: | |
28 | + dat = {} | |
29 | + info = {} | |
30 | + | |
31 | + if float(table.extent.split(",")[0]) < 180: | |
30 | 32 | grid_size = copy.copy(lonlat_gridsize) |
31 | 33 | else: |
32 | 34 | grid_size = copy.copy(project_gridsize) |
... | ... | @@ -36,12 +38,13 @@ class Api(ApiTemplate): |
36 | 38 | grid_size.reverse() |
37 | 39 | |
38 | 40 | # 正在精华的任务 |
39 | - task_running = Task.query.filter_by(task_type=2,table_guid=table_guid,state=0).all() | |
41 | + task_running = Task.query.filter_by(task_type=2, table_guid=table_guid, state=0).all() | |
40 | 42 | grid_running = [] |
41 | - | |
43 | + | |
42 | 44 | for t in task_running: |
43 | 45 | grid_running.extend(t.parameter.split(",")) |
44 | 46 | |
47 | + info["type"] = "m" | |
45 | 48 | for grid in grid_size: |
46 | 49 | if grid in now_grid: |
47 | 50 | dat[str(grid)] = 1 |
... | ... | @@ -49,8 +52,14 @@ class Api(ApiTemplate): |
49 | 52 | dat[str(grid)] = 0 |
50 | 53 | if str(grid) in grid_running: |
51 | 54 | dat[str(grid)] = 2 |
55 | + # 特殊处理一下 | |
56 | + if grid == 0.00008: | |
57 | + dat["0.00008"] = dat[str(grid)] | |
58 | + del dat[str(grid)] | |
59 | + info["type"] = "d" | |
60 | + info["info"] = dat | |
52 | 61 | |
53 | - res["data"] = json.dumps(dat) | |
62 | + res["data"] = info | |
54 | 63 | res["result"] = True |
55 | 64 | |
56 | 65 | except Exception as e: | ... | ... |
... | ... | @@ -121,13 +121,16 @@ class Api(ApiTemplate): |
121 | 121 | |
122 | 122 | pg_ds :DataSource= PGUtil.open_pg_data_source(1,DES.decode(database.sqlalchemy_uri)) |
123 | 123 | |
124 | + | |
125 | + | |
126 | + | |
124 | 127 | # 创建抽稀过程 |
125 | 128 | options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name,pg_session)), |
126 | 129 | "PRECISION=NO"] |
127 | 130 | |
128 | 131 | layer = pg_ds.GetLayerByName(table.name) |
129 | 132 | |
130 | - vacuate_process:VacuateProcess = VacuateProcess(layer, table.name, options,database.sqlalchemy_uri,grids) | |
133 | + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database.sqlalchemy_uri,grids) | |
131 | 134 | |
132 | 135 | |
133 | 136 | for feature in layer: |
... | ... | @@ -233,7 +236,7 @@ class VacuateProcess: |
233 | 236 | this_gridsize=[] |
234 | 237 | |
235 | 238 | |
236 | - def __init__(self,layer:Layer,new_layer_name, options,sqlalchemy_uri,grids): | |
239 | + def __init__(self,layer:Layer,table_guid, options,sqlalchemy_uri,grids): | |
237 | 240 | |
238 | 241 | #是空间图层才初始化 |
239 | 242 | if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: |
... | ... | @@ -254,7 +257,9 @@ class VacuateProcess: |
254 | 257 | |
255 | 258 | # 创建抽稀ds |
256 | 259 | for l in range(self.max_level): |
257 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
260 | + | |
261 | + pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.SQLALCHEMY_DATABASE_URI) | |
262 | + # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
258 | 263 | pg_ds_l.StartTransaction() |
259 | 264 | self.pg_ds_dict[l] = pg_ds_l |
260 | 265 | |
... | ... | @@ -271,8 +276,8 @@ class VacuateProcess: |
271 | 276 | |
272 | 277 | if this_grid_len<1: |
273 | 278 | grid_name = str(this_grid_len).split(".")[-1] |
274 | - if this_grid_len.__eq__(0.000075): | |
275 | - grid_name = "000075" | |
279 | + if this_grid_len.__eq__(0.00008): | |
280 | + grid_name = "00008" | |
276 | 281 | else: |
277 | 282 | grid_name = str(int(this_grid_len)) |
278 | 283 | |
... | ... | @@ -284,8 +289,10 @@ class VacuateProcess: |
284 | 289 | |
285 | 290 | lev = this_grid_szie.index(this_grid_len) |
286 | 291 | print("{}:{}".format(grid_name, lev)) |
287 | - vl = pg.CreateLayer("{}_vacuate_{}_{}".format(new_layer_name, lev, grid_name), layer.GetSpatialRef(), | |
288 | - ogr.wkbUnknown, options) | |
292 | + | |
293 | + | |
294 | + v_ln = "z{}_vacuate_{}_{}".format(table_guid,lev, grid_name) | |
295 | + vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | |
289 | 296 | # 抽稀表不需要属性 |
290 | 297 | # vl.CreateFields(schema) |
291 | 298 | self.vacuate_layers[l] = vl |
... | ... | @@ -308,6 +315,13 @@ class VacuateProcess: |
308 | 315 | lat_extent = extent[3]-extent[2] |
309 | 316 | |
310 | 317 | this_grid_len =self.vacuate_layers_gridsize[level] |
318 | + #超大的直接加入 | |
319 | + # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | |
320 | + # vacuate_layer: Layer = self.vacuate_layers.get(level) | |
321 | + # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
322 | + # feat.SetGeometry(g) | |
323 | + # vacuate_layer.CreateFeature(feat) | |
324 | + # else: | |
311 | 325 | |
312 | 326 | row = int((center.GetY() - self.extent[2]) / this_grid_len) |
313 | 327 | col = int((center.GetX() - self.extent[0]) / this_grid_len) |
... | ... | @@ -325,7 +339,14 @@ class VacuateProcess: |
325 | 339 | else: |
326 | 340 | feat.SetGeometry(g) |
327 | 341 | vacuate_layer.CreateFeature(feat) |
328 | - self.fill_dict[key] += 1 | |
342 | + self.fill_dict[key] += 1 | |
343 | + #超大的还有机会 | |
344 | + elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | |
345 | + vacuate_layer: Layer = self.vacuate_layers.get(level) | |
346 | + feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
347 | + feat.SetGeometry(g) | |
348 | + vacuate_layer.CreateFeature(feat) | |
349 | + self.fill_dict[key] += 1 | |
329 | 350 | |
330 | 351 | def end(self): |
331 | 352 | for pg in self.pg_ds_dict.values(): | ... | ... |
... | ... | @@ -43,10 +43,6 @@ class Api(ApiTemplate): |
43 | 43 | layer = pg_ds.GetLayerByName(table.name) |
44 | 44 | |
45 | 45 | # 判断用户权限 |
46 | - user, pw, host, port, database = PGUtil.get_info_from_sqlachemy_uri(DES.decode(table.relate_database.sqlalchemy_uri)) | |
47 | - query_role:Layer = pg_ds.ExecuteSQL("SELECT rolcreatedb FROM pg_roles WHERE rolname='{}'".format(user)) | |
48 | - if not query_role.GetFeature(0).GetField("rolcreatedb"): | |
49 | - raise Exception("用户不具备建表权限!") | |
50 | 46 | |
51 | 47 | if not layer: |
52 | 48 | raise Exception("图层不存在!") | ... | ... |
app/util/component/CopyData2.py
0 → 100644
1 | +# coding=utf-8 | |
2 | +#author: 4N | |
3 | +#createtime: 2021/6/11 | |
4 | +#email: nheweijun@sina.com | |
5 | +import copy | |
6 | +from osgeo.ogr import * | |
7 | +from osgeo import gdal,ogr | |
8 | +import uuid | |
9 | +import time | |
10 | +import os | |
11 | +import json | |
12 | + | |
13 | + | |
14 | +def get_info_from_sqlachemy_uri(uri): | |
15 | + parts = uri.split(":") | |
16 | + user = parts[1][2:] | |
17 | + | |
18 | + password_list = parts[2].split("@") | |
19 | + if password_list.__len__() > 2: | |
20 | + password = "@".join(password_list[:-1]) | |
21 | + else: | |
22 | + password = parts[2].split("@")[0] | |
23 | + host = parts[2].split("@")[-1] | |
24 | + port = parts[3].split("/")[0] | |
25 | + database = parts[3].split("/")[1] | |
26 | + | |
27 | + return user, password, host, port, database | |
28 | + | |
29 | +def open_pg_data_source(iswrite, uri): | |
30 | + """ | |
31 | + # 获取PostGIS数据源 | |
32 | + :return: | |
33 | + """ | |
34 | + db_conn_tuple = get_info_from_sqlachemy_uri(uri) | |
35 | + fn = "PG: user=%s password=%s host=%s port=%s dbname=%s " % db_conn_tuple | |
36 | + driver = ogr.GetDriverByName("PostgreSQL") | |
37 | + if driver is None: | |
38 | + raise Exception("打开PostgreSQL驱动失败,可能是当前GDAL未支持PostgreSQL驱动!") | |
39 | + ds = driver.Open(fn, iswrite) | |
40 | + if ds is None: | |
41 | + raise Exception("打开数据源失败!") | |
42 | + return ds | |
43 | + | |
44 | + | |
45 | +def move_digui(coords,offx,offy): | |
46 | + if isinstance(coords[0],list): | |
47 | + for coord in coords: | |
48 | + move_digui(coord, offx, offy) | |
49 | + else : | |
50 | + coords[0] += offx | |
51 | + coords[1] += offy | |
52 | + | |
53 | +def move(geo,offx,offy): | |
54 | + | |
55 | + | |
56 | + geojson_str = geo.ExportToJson() | |
57 | + geojson = json.loads(geojson_str) | |
58 | + | |
59 | + coords = geojson["coordinates"] | |
60 | + move_digui(coords, offx, offy) | |
61 | + og = ogr.CreateGeometryFromJson(json.dumps(geojson)) | |
62 | + | |
63 | + return og | |
64 | + | |
65 | + | |
66 | +def copydata(bound,input,output): | |
67 | + work_dir = os.path.dirname(os.path.abspath(__file__)) | |
68 | + | |
69 | + shp_driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | |
70 | + bound_ds : DataSource = shp_driver.Open(bound, 0) | |
71 | + bound_layer :Layer = bound_ds.GetLayer(0) | |
72 | + | |
73 | + bf = bound_layer.GetNextFeature() | |
74 | + bound_geom = bf.GetGeometryRef() | |
75 | + | |
76 | + | |
77 | + | |
78 | + gdb_driver: Driver = ogr.GetDriverByName("FileGDB") | |
79 | + # gdb_ds: DataSource = gdb_driver.Open(gdb, 0) | |
80 | + # layer: Layer = gdb_ds.GetLayerByName(layer_name) | |
81 | + | |
82 | + input_ds: DataSource = shp_driver.Open(input, 0) | |
83 | + layer: Layer = input_ds.GetLayer(0) | |
84 | + | |
85 | + if not layer: | |
86 | + raise Exception("打开数据失败!") | |
87 | + | |
88 | + | |
89 | + output_ds: DataSource = gdb_driver.CreateDataSource(output) | |
90 | + | |
91 | + | |
92 | + gdb_layer:Layer = output_ds.CreateLayer(layer.GetName(),layer.GetSpatialRef(),layer.GetGeomType()) | |
93 | + | |
94 | + gdb_layer.CreateFields(layer.schema) | |
95 | + | |
96 | + | |
97 | + | |
98 | + | |
99 | + # | |
100 | + bound_exent = bound_layer.GetExtent() | |
101 | + | |
102 | + layer_extent = layer.GetExtent() | |
103 | + | |
104 | + xxrange=layer_extent[1]-layer_extent[0] | |
105 | + yrange = layer_extent[3]-layer_extent[2] | |
106 | + | |
107 | + | |
108 | + left = (int((layer_extent[0] - bound_exent[0]) / xxrange) + 1) * (-1) | |
109 | + right = int((bound_exent[1] - layer_extent[1]) / xxrange) + 1 | |
110 | + up = int((bound_exent[3] - layer_extent[3]) / yrange) + 1 | |
111 | + down = (int((layer_extent[2] - bound_exent[2]) / yrange) + 1) * (-1) | |
112 | + | |
113 | + | |
114 | + begin = time.time() | |
115 | + count=1 | |
116 | + | |
117 | + | |
118 | + for f in layer: | |
119 | + | |
120 | + new_f = copy.copy(f) | |
121 | + for xt in range(left,right+1,1): | |
122 | + for yt in range(down,up+1,1): | |
123 | + out_g = move(f.GetGeometryRef(),xxrange*xt,yrange*yt) | |
124 | + | |
125 | + if out_g.Intersect(bound_geom): | |
126 | + out_g = out_g.Intersection(bound_geom) | |
127 | + new_f.SetGeometry(out_g) | |
128 | + new_f.SetFID(count) | |
129 | + gdb_layer.CreateFeature(new_f) | |
130 | + count += 1 | |
131 | + if count % 10000 == 0: | |
132 | + # print(count) | |
133 | + with open(os.path.join(work_dir, "copy.txt"), "w") as fi: | |
134 | + fi.write("已完成{}".format(count)) | |
135 | + | |
136 | + print(time.time()-begin) | |
137 | + | |
138 | + input_ds.Destroy() | |
139 | + | |
140 | +def envelop_2_polygon(env): | |
141 | + ring = ogr.Geometry(ogr.wkbLinearRing) | |
142 | + ring.AddPoint(env[0], env[2]) | |
143 | + ring.AddPoint(env[0], env[3]) | |
144 | + ring.AddPoint(env[1], env[3]) | |
145 | + ring.AddPoint(env[1], env[2]) | |
146 | + ring.AddPoint(env[0], env[2]) | |
147 | + # Create polygon | |
148 | + poly = ogr.Geometry(ogr.wkbPolygon) | |
149 | + poly.AddGeometry(ring) | |
150 | + return poly | |
151 | + | |
152 | +def clip(): | |
153 | + | |
154 | + input = r"E:\Data\copy\x.shp" | |
155 | + output = r"E:\Data\copy\xianorigin.shp" | |
156 | + gdal.SetConfigOption("SHAPE_ENCODING", "UTF-8") | |
157 | + shp_driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | |
158 | + | |
159 | + input_ds: DataSource = shp_driver.Open(input, 0) | |
160 | + layer: Layer = input_ds.GetLayer(0) | |
161 | + | |
162 | + bound_geo = envelop_2_polygon([112.84,112.952,22.914,22.985]) | |
163 | + | |
164 | + if not layer: | |
165 | + raise Exception("打开数据失败!") | |
166 | + | |
167 | + output_ds: DataSource = shp_driver.CreateDataSource(output) | |
168 | + | |
169 | + out_layer: Layer = output_ds.CreateLayer(layer.GetName(), layer.GetSpatialRef(), layer.GetGeomType()) | |
170 | + out_layer.CreateFields(layer.schema) | |
171 | + for f in layer: | |
172 | + new_f = copy.copy(f) | |
173 | + out_g = f.GetGeometryRef() | |
174 | + if out_g.Intersect(bound_geo): | |
175 | + out_g = out_g.Intersection(bound_geo) | |
176 | + new_f.SetGeometry(out_g) | |
177 | + # new_f.UnsetField("OBJECTID") | |
178 | + out_layer.CreateFeature(new_f) | |
179 | + | |
180 | + output_ds.Destroy() | |
181 | + | |
182 | + | |
183 | + | |
184 | +if __name__ == '__main__': | |
185 | + # bound_shp = "E:\Data\广东边界\gdsample.shp" | |
186 | + # input = r"E:\Data\copy\origin.shp" | |
187 | + # output = r"E:\Data\copy\re.shp" | |
188 | + # copydata(bound_shp,input,output) | |
189 | + | |
190 | + # bound_shp = "/root/CopyData/gdsample.shp" | |
191 | + # input = r"/root/CopyData/origin.shp" | |
192 | + # output = r"/root/CopyData/re.gdb" | |
193 | + # copydata(bound_shp,input,output) | |
194 | + clip() | |
195 | + | |
196 | + | |
197 | + | |
198 | + | |
199 | + | |
200 | + | |
201 | + | |
202 | + | |
203 | + | |
204 | + | |
205 | + | ... | ... |
... | ... | @@ -159,7 +159,7 @@ class EntryDataVacuate: |
159 | 159 | new_layer_name = None |
160 | 160 | vacuate_process= None |
161 | 161 | success = True |
162 | - | |
162 | + table_guid = uuid.uuid1().__str__() | |
163 | 163 | try: |
164 | 164 | # 图层设置 |
165 | 165 | parameter = this_task.parameter |
... | ... | @@ -200,7 +200,7 @@ class EntryDataVacuate: |
200 | 200 | pg_layer.CreateFields(schema) |
201 | 201 | |
202 | 202 | #创建抽稀过程 |
203 | - vacuate_process = VacuateProcess(layer,new_layer_name,options,this_task.database.sqlalchemy_uri) | |
203 | + vacuate_process = VacuateProcess(layer,table_guid,options) | |
204 | 204 | |
205 | 205 | |
206 | 206 | |
... | ... | @@ -235,7 +235,8 @@ class EntryDataVacuate: |
235 | 235 | # 注册图层信息 |
236 | 236 | # 是否抽吸过 |
237 | 237 | is_vacuate = 1 if vacuate_process.max_level>0 else 0 |
238 | - table_guid = this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate) | |
238 | + | |
239 | + this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | |
239 | 240 | |
240 | 241 | # 注册抽稀表 |
241 | 242 | this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) |
... | ... | @@ -295,7 +296,7 @@ class ThisTask: |
295 | 296 | self.process_session.add(task_process) |
296 | 297 | self.process_session.commit() |
297 | 298 | |
298 | - def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate): | |
299 | + def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate,table_guid): | |
299 | 300 | ''' |
300 | 301 | 注册表 |
301 | 302 | :param layer: 图层 |
... | ... | @@ -321,7 +322,7 @@ class ThisTask: |
321 | 322 | geom_type = GeometryAdapter.get_geometry_type(layer) |
322 | 323 | |
323 | 324 | extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) |
324 | - table_guid = uuid.uuid1().__str__() | |
325 | + | |
325 | 326 | table = Table(guid=table_guid, |
326 | 327 | database_guid=self.database.guid, |
327 | 328 | creator=creator, |
... | ... | @@ -419,7 +420,7 @@ class VacuateProcess: |
419 | 420 | this_gridsize=[] |
420 | 421 | |
421 | 422 | |
422 | - def __init__(self,layer:Layer,new_layer_name, options,sqlalchemy_uri): | |
423 | + def __init__(self,layer:Layer,table_guid, options): | |
423 | 424 | |
424 | 425 | #是空间图层才初始化 |
425 | 426 | if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: |
... | ... | @@ -466,6 +467,7 @@ class VacuateProcess: |
466 | 467 | self.t_grid_size=self.project_gridsize |
467 | 468 | else: |
468 | 469 | self.t_grid_size = self.lonlat_gridsize |
470 | + | |
469 | 471 | for grid_size in self.t_grid_size: |
470 | 472 | # 最少抽稀个数 |
471 | 473 | if lc > self.least_vacuate_count: |
... | ... | @@ -482,7 +484,8 @@ class VacuateProcess: |
482 | 484 | |
483 | 485 | # 创建抽稀ds |
484 | 486 | for l in range(self.max_level): |
485 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
487 | + # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
488 | + pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.SQLALCHEMY_DATABASE_URI) | |
486 | 489 | pg_ds_l.StartTransaction() |
487 | 490 | self.pg_ds_dict[l] = pg_ds_l |
488 | 491 | |
... | ... | @@ -500,16 +503,19 @@ class VacuateProcess: |
500 | 503 | grid_name = str(this_grid_len) |
501 | 504 | if this_grid_len<1: |
502 | 505 | grid_name = str(this_grid_len).split(".")[-1] |
503 | - if this_grid_len.__eq__(0.000075): | |
504 | - grid_name = "000075" | |
506 | + if this_grid_len.__eq__(0.00008): | |
507 | + grid_name = "00008" | |
505 | 508 | |
506 | 509 | # 抽稀图层是点面混合的 |
507 | 510 | # 抽稀表有固定的命名规则 |
508 | 511 | # 抽稀表一定要覆盖 |
509 | 512 | |
510 | 513 | |
511 | - vl = pg.CreateLayer("{}_vacuate_{}_{}".format(new_layer_name, self.t_grid_size.index(this_grid_len), grid_name), layer.GetSpatialRef(), | |
512 | - ogr.wkbUnknown, options) | |
514 | + print("{}:{}".format(self.t_grid_size.index(this_grid_len),this_grid_len)) | |
515 | + | |
516 | + | |
517 | + v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name) | |
518 | + vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | |
513 | 519 | # 抽稀表不需要属性 |
514 | 520 | # vl.CreateFields(schema) |
515 | 521 | self.vacuate_layers[l] = vl |
... | ... | @@ -532,6 +538,13 @@ class VacuateProcess: |
532 | 538 | lat_extent = extent[3]-extent[2] |
533 | 539 | |
534 | 540 | this_grid_len =self.vacuate_layers_gridsize[level] |
541 | + #超大的直接加入 | |
542 | + # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | |
543 | + # vacuate_layer: Layer = self.vacuate_layers.get(level) | |
544 | + # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
545 | + # feat.SetGeometry(g) | |
546 | + # vacuate_layer.CreateFeature(feat) | |
547 | + # else: | |
535 | 548 | |
536 | 549 | row = int((center.GetY() - self.extent[2]) / this_grid_len) |
537 | 550 | col = int((center.GetX() - self.extent[0]) / this_grid_len) |
... | ... | @@ -549,7 +562,14 @@ class VacuateProcess: |
549 | 562 | else: |
550 | 563 | feat.SetGeometry(g) |
551 | 564 | vacuate_layer.CreateFeature(feat) |
552 | - self.fill_dict[key] += 1 | |
565 | + self.fill_dict[key] += 1 | |
566 | + #超大的还有机会 | |
567 | + elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | |
568 | + vacuate_layer: Layer = self.vacuate_layers.get(level) | |
569 | + feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
570 | + feat.SetGeometry(g) | |
571 | + vacuate_layer.CreateFeature(feat) | |
572 | + self.fill_dict[key] += 1 | |
553 | 573 | |
554 | 574 | def end(self): |
555 | 575 | for pg in self.pg_ds_dict.values(): | ... | ... |
app/util/component/copy.txt
0 → 100644
1 | +メムヘノ20000 | |
\ No newline at end of file | ... | ... |
... | ... | @@ -355,6 +355,9 @@ IncludeOptional conf.d/*.conf |
355 | 355 | LoadModule wsgi_module "/usr/lib64/httpd/modules/mod_wsgi-py37.cpython-37m-x86_64-linux-gnu.so" |
356 | 356 | "/var/gdal" |
357 | 357 | |
358 | + dmapmanager processes=4 threads=16 display-name=%{GROUP} | |
359 | + dmapmanager | |
360 | + | |
358 | 361 | / /usr/src/app/run.wsgi |
359 | 362 | <Directory /usr/> |
360 | 363 | Require all granted | ... | ... |
... | ... | @@ -19,15 +19,5 @@ import json |
19 | 19 | # print(sys.getsizeof(json.dumps(pixel_array.tolist()))) |
20 | 20 | # print(sys.getsizeof((content))) |
21 | 21 | |
22 | -def c(dd): | |
23 | - return 1 | |
24 | - | |
25 | -dd = numpy.zeros((2000, 2000), dtype=int) + 2 | |
26 | - | |
27 | - | |
28 | -dd[range(0, 5), 1]=1 | |
29 | - | |
30 | - | |
31 | - | |
32 | - | |
33 | -print(dd) | |
\ No newline at end of file | ||
22 | +for x in range(-3,3,1): | |
23 | + print(x) | |
\ No newline at end of file | ... | ... |
请
注册
或
登录
后发表评论