正在显示
14 个修改的文件
包含
341 行增加
和
80 行删除
| ... | ... | @@ -4,29 +4,51 @@ |
| 4 | 4 | #email: nheweijun@sina.com |
| 5 | 5 | |
| 6 | 6 | |
| 7 | -from app.models import Database,db | |
| 7 | +from app.models import Database,db,Table,TableVacuate | |
| 8 | 8 | |
| 9 | 9 | |
| 10 | 10 | from app.util.component.ApiTemplate import ApiTemplate |
| 11 | +from app.util.component.PGUtil import PGUtil | |
| 12 | +import configure | |
| 13 | +from osgeo.ogr import DataSource | |
| 14 | +from flask import current_app | |
| 11 | 15 | class Api(ApiTemplate): |
| 12 | 16 | api_name = "删除数据库" |
| 13 | 17 | def process(self): |
| 14 | - res ={} | |
| 18 | + re ={} | |
| 19 | + sys_ds = None | |
| 15 | 20 | try: |
| 16 | 21 | |
| 17 | 22 | database = db.session.query(Database).filter_by(guid=self.para.get("guid")).one_or_none() |
| 23 | + sys_ds :DataSource= PGUtil.open_pg_data_source(1,configure.SQLALCHEMY_DATABASE_URI) | |
| 18 | 24 | if database: |
| 25 | + # 删除table | |
| 26 | + re_tables = Table.query.filter_by(database_guid=database.guid).all() | |
| 27 | + for table in re_tables: | |
| 28 | + re_tvs = TableVacuate.query.filter_by(table_guid=table.guid).all() | |
| 29 | + for tv in re_tvs: | |
| 30 | + db.session.delete(tv) | |
| 31 | + try: | |
| 32 | + sys_ds.DeleteLayer(tv.name) | |
| 33 | + except: | |
| 34 | + current_app.logger.warning("{}不存在".format(tv.name)) | |
| 35 | + db.session.delete(table) | |
| 19 | 36 | db.session.delete(database) |
| 37 | + | |
| 20 | 38 | db.session.commit() |
| 21 | - res["msg"] = "数据库删除成功!" | |
| 22 | - res["result"] = True | |
| 39 | + re["result"] = True | |
| 40 | + re["msg"] = "数据库删除成功!" | |
| 23 | 41 | else: |
| 24 | - res["msg"] = "数据库不存在!" | |
| 25 | - res["result"] = False | |
| 42 | + re["result"] = False | |
| 43 | + re["msg"] = "数据库不存在!" | |
| 26 | 44 | except Exception as e: |
| 27 | 45 | db.session.rollback() |
| 46 | + sys_ds.RollbackTransaction() | |
| 28 | 47 | raise e |
| 29 | - return res | |
| 48 | + finally: | |
| 49 | + if sys_ds: | |
| 50 | + sys_ds.Destroy() | |
| 51 | + return re | |
| 30 | 52 | |
| 31 | 53 | |
| 32 | 54 | ... | ... |
| ... | ... | @@ -17,6 +17,7 @@ from app.util.component.PGUtil import PGUtil |
| 17 | 17 | from app.util.component.SQLUtil import SQLUtil |
| 18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
| 19 | 19 | from app.util.component.StructuredPrint import StructurePrint |
| 20 | +import configure | |
| 20 | 21 | class Api(ApiTemplate): |
| 21 | 22 | api_name = "注册数据库" |
| 22 | 23 | def process(self): |
| ... | ... | @@ -57,7 +58,6 @@ class Api(ApiTemplate): |
| 57 | 58 | elif not self.check_space(sqlalchemy_uri): |
| 58 | 59 | res["msg"] = "数据不是空间数据库!" |
| 59 | 60 | return res |
| 60 | - | |
| 61 | 61 | else: |
| 62 | 62 | this_time = datetime.datetime.now() |
| 63 | 63 | database_guid = uuid.uuid1().__str__() |
| ... | ... | @@ -96,12 +96,8 @@ class Api(ApiTemplate): |
| 96 | 96 | # 注册空间表 |
| 97 | 97 | spatial_table_name,tables = self.register_spatial_table(pg_ds, database, this_time,db_tuple) |
| 98 | 98 | |
| 99 | - #注册抽稀表 | |
| 100 | - self.regiser_vacuate_table(pg_ds,tables,db_tuple) | |
| 101 | - | |
| 102 | 99 | #注册普通表 |
| 103 | 100 | self.register_common_table(this_time,database,spatial_table_name,db_tuple) |
| 104 | - | |
| 105 | 101 | pg_ds.Destroy() |
| 106 | 102 | |
| 107 | 103 | |
| ... | ... | @@ -245,11 +241,11 @@ class Api(ApiTemplate): |
| 245 | 241 | if l_name.__contains__("_vacuate_"): |
| 246 | 242 | |
| 247 | 243 | # 没有权限的表跳过 |
| 248 | - if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
| 249 | - StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
| 250 | - continue | |
| 244 | + # if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
| 245 | + # StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
| 246 | + # continue | |
| 251 | 247 | |
| 252 | - base_layer_name=l_name.split("_vacuate_")[0] | |
| 248 | + base_layer_name=l_name.split("_vacuate_")[1] | |
| 253 | 249 | level = l_name.split("_")[-2] |
| 254 | 250 | pixel_distance_str: str ="0" |
| 255 | 251 | try: | ... | ... |
| ... | ... | @@ -15,7 +15,6 @@ import uuid |
| 15 | 15 | from sqlalchemy.orm import Session |
| 16 | 16 | from app.util.component.SQLUtil import SQLUtil |
| 17 | 17 | from app.util.component.PGUtil import PGUtil |
| 18 | -from app.util.component.ModelVisitor import ModelVisitor | |
| 19 | 18 | from app.util.component.StructuredPrint import StructurePrint |
| 20 | 19 | from app.util.component.ApiTemplate import ApiTemplate |
| 21 | 20 | from app.util.component.GeometryAdapter import GeometryAdapter |
| ... | ... | @@ -37,8 +36,6 @@ class Api(ApiTemplate): |
| 37 | 36 | # 初始化task |
| 38 | 37 | task_guid = uuid.uuid1().__str__() |
| 39 | 38 | |
| 40 | - # result = self.table_refresh(database) | |
| 41 | - | |
| 42 | 39 | refresh_process = multiprocessing.Process(target=self.table_refresh,args=(database,task_guid)) |
| 43 | 40 | refresh_process.start() |
| 44 | 41 | |
| ... | ... | @@ -67,6 +64,7 @@ class Api(ApiTemplate): |
| 67 | 64 | def table_refresh(self,database,task_guid): |
| 68 | 65 | |
| 69 | 66 | pg_ds =None |
| 67 | + sys_ds =None | |
| 70 | 68 | data_session=None |
| 71 | 69 | result = {} |
| 72 | 70 | sys_session = None |
| ... | ... | @@ -74,7 +72,7 @@ class Api(ApiTemplate): |
| 74 | 72 | |
| 75 | 73 | try: |
| 76 | 74 | sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI) |
| 77 | - | |
| 75 | + sys_ds = PGUtil.open_pg_data_source(0,configure.SQLALCHEMY_DATABASE_URI) | |
| 78 | 76 | |
| 79 | 77 | this_time = datetime.datetime.now() |
| 80 | 78 | database_guid = database.guid |
| ... | ... | @@ -99,8 +97,6 @@ class Api(ApiTemplate): |
| 99 | 97 | # 空间表处理完毕 |
| 100 | 98 | sys_session.commit() |
| 101 | 99 | |
| 102 | - # 处理抽稀表 | |
| 103 | - self.deal_vacuate_table(pg_ds,sys_session, database.guid,db_tuple) | |
| 104 | 100 | |
| 105 | 101 | # 空间表处理完毕 |
| 106 | 102 | sys_session.commit() |
| ... | ... | @@ -168,6 +164,8 @@ class Api(ApiTemplate): |
| 168 | 164 | data_session.close() |
| 169 | 165 | if sys_session: |
| 170 | 166 | sys_session.close() |
| 167 | + if sys_ds: | |
| 168 | + sys_ds.Destroy() | |
| 171 | 169 | return result |
| 172 | 170 | |
| 173 | 171 | def add_spatail_table(self,database,pg_ds,sys_session,spatial_tables_names,this_time,db_tuple): |
| ... | ... | @@ -251,11 +249,11 @@ class Api(ApiTemplate): |
| 251 | 249 | sys_session.add(column) |
| 252 | 250 | return db_tables_names |
| 253 | 251 | |
| 254 | - def deal_vacuate_table(self,pg_ds,sys_session,database_guid,db_tuple): | |
| 252 | + def deal_vacuate_table(self,sys_ds,sys_session,database_guid): | |
| 255 | 253 | |
| 256 | 254 | |
| 257 | - for i in range(pg_ds.GetLayerCount()): | |
| 258 | - layer: Layer = pg_ds.GetLayer(i) | |
| 255 | + for i in range(sys_ds.GetLayerCount()): | |
| 256 | + layer: Layer = sys_ds.GetLayer(i) | |
| 259 | 257 | geom_column = layer.GetGeometryColumn() |
| 260 | 258 | |
| 261 | 259 | if not geom_column: |
| ... | ... | @@ -266,12 +264,8 @@ class Api(ApiTemplate): |
| 266 | 264 | if layer.GetName().__contains__("_vacuate_"): |
| 267 | 265 | l_name = layer.GetName() |
| 268 | 266 | |
| 269 | - # 没有权限的表跳过 | |
| 270 | - if not PGUtil.check_table_privilege(l_name, "SELECT", db_tuple[0], pg_ds): | |
| 271 | - StructurePrint.print("用户{}对表{}没有select权限!".format(db_tuple[0], l_name), "warn") | |
| 272 | - continue | |
| 267 | + base_layer_name = l_name.split("_vacuate_")[0].split("_")[1] | |
| 273 | 268 | |
| 274 | - base_layer_name = l_name.split("_vacuate_")[0] | |
| 275 | 269 | level = l_name.split("_")[-2] |
| 276 | 270 | |
| 277 | 271 | pixel_distance_str: str ="0" | ... | ... |
| ... | ... | @@ -132,7 +132,7 @@ class Api(ApiTemplate): |
| 132 | 132 | |
| 133 | 133 | layer = pg_ds.GetLayerByName(table.name) |
| 134 | 134 | |
| 135 | - vacuate_process:VacuateProcess = VacuateProcess(layer, table.name, options,database.sqlalchemy_uri) | |
| 135 | + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options) | |
| 136 | 136 | |
| 137 | 137 | |
| 138 | 138 | for feature in layer: | ... | ... |
| ... | ... | @@ -18,6 +18,7 @@ import uuid |
| 18 | 18 | import configure |
| 19 | 19 | from osgeo.ogr import DataSource,Layer,Geometry |
| 20 | 20 | from osgeo import ogr |
| 21 | +from flask import current_app | |
| 21 | 22 | |
| 22 | 23 | class Api(ApiTemplate): |
| 23 | 24 | api_name = "删除抽稀表" |
| ... | ... | @@ -26,6 +27,7 @@ class Api(ApiTemplate): |
| 26 | 27 | res = {} |
| 27 | 28 | res["data"] = {} |
| 28 | 29 | pg_ds = None |
| 30 | + sys_ds = None | |
| 29 | 31 | try: |
| 30 | 32 | table_guid = self.para.get("guid") |
| 31 | 33 | table: Table = Table.query.filter_by(guid=table_guid).one_or_none() |
| ... | ... | @@ -40,13 +42,9 @@ class Api(ApiTemplate): |
| 40 | 42 | |
| 41 | 43 | # 判断图层是否存在 |
| 42 | 44 | pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(table.relate_database.sqlalchemy_uri)) |
| 43 | - layer = pg_ds.GetLayerByName(table.name) | |
| 45 | + sys_ds :DataSource= PGUtil.open_pg_data_source(0,configure.SQLALCHEMY_DATABASE_URI) | |
| 44 | 46 | |
| 45 | - # 判断用户权限 | |
| 46 | - # user, pw, host, port, database = PGUtil.get_info_from_sqlachemy_uri(DES.decode(table.relate_database.sqlalchemy_uri)) | |
| 47 | - # query_role:Layer = pg_ds.ExecuteSQL("SELECT rolcreatedb FROM pg_roles WHERE rolname='{}'".format(user)) | |
| 48 | - # if not query_role.GetFeature(0).GetField("rolcreatedb"): | |
| 49 | - # raise Exception("用户不具备建表权限!") | |
| 47 | + layer = pg_ds.GetLayerByName(table.name) | |
| 50 | 48 | |
| 51 | 49 | if not layer: |
| 52 | 50 | raise Exception("图层不存在!") |
| ... | ... | @@ -58,9 +56,9 @@ class Api(ApiTemplate): |
| 58 | 56 | for tv in tvs : |
| 59 | 57 | db.session.delete(tv) |
| 60 | 58 | try: |
| 61 | - pg_ds.DeleteLayer(tv.name) | |
| 59 | + sys_ds.DeleteLayer(tv.name) | |
| 62 | 60 | except Exception as e : |
| 63 | - StructurePrint.print("抽稀图层不存在!","warn") | |
| 61 | + current_app.logger.warning("抽稀图层不存在!") | |
| 64 | 62 | db.session.commit() |
| 65 | 63 | res["result"] = True |
| 66 | 64 | |
| ... | ... | @@ -69,6 +67,8 @@ class Api(ApiTemplate): |
| 69 | 67 | finally: |
| 70 | 68 | if pg_ds: |
| 71 | 69 | pg_ds.Destroy() |
| 70 | + if sys_ds: | |
| 71 | + sys_ds.Destroy() | |
| 72 | 72 | return res |
| 73 | 73 | |
| 74 | 74 | ... | ... |
| ... | ... | @@ -25,8 +25,10 @@ class Api(ApiTemplate): |
| 25 | 25 | if not table: |
| 26 | 26 | raise Exception("数据不存在!") |
| 27 | 27 | tvs = TableVacuate.query.filter_by(table_guid=table_guid).all() |
| 28 | - dat = collections.OrderedDict() | |
| 29 | - if float(table.extent.split(",")[0])<180: | |
| 28 | + dat = {} | |
| 29 | + info = {} | |
| 30 | + | |
| 31 | + if float(table.extent.split(",")[0]) < 180: | |
| 30 | 32 | grid_size = copy.copy(lonlat_gridsize) |
| 31 | 33 | else: |
| 32 | 34 | grid_size = copy.copy(project_gridsize) |
| ... | ... | @@ -36,12 +38,13 @@ class Api(ApiTemplate): |
| 36 | 38 | grid_size.reverse() |
| 37 | 39 | |
| 38 | 40 | # 正在精华的任务 |
| 39 | - task_running = Task.query.filter_by(task_type=2,table_guid=table_guid,state=0).all() | |
| 41 | + task_running = Task.query.filter_by(task_type=2, table_guid=table_guid, state=0).all() | |
| 40 | 42 | grid_running = [] |
| 41 | - | |
| 43 | + | |
| 42 | 44 | for t in task_running: |
| 43 | 45 | grid_running.extend(t.parameter.split(",")) |
| 44 | 46 | |
| 47 | + info["type"] = "m" | |
| 45 | 48 | for grid in grid_size: |
| 46 | 49 | if grid in now_grid: |
| 47 | 50 | dat[str(grid)] = 1 |
| ... | ... | @@ -49,8 +52,14 @@ class Api(ApiTemplate): |
| 49 | 52 | dat[str(grid)] = 0 |
| 50 | 53 | if str(grid) in grid_running: |
| 51 | 54 | dat[str(grid)] = 2 |
| 55 | + # 特殊处理一下 | |
| 56 | + if grid == 0.00008: | |
| 57 | + dat["0.00008"] = dat[str(grid)] | |
| 58 | + del dat[str(grid)] | |
| 59 | + info["type"] = "d" | |
| 60 | + info["info"] = dat | |
| 52 | 61 | |
| 53 | - res["data"] = json.dumps(dat) | |
| 62 | + res["data"] = info | |
| 54 | 63 | res["result"] = True |
| 55 | 64 | |
| 56 | 65 | except Exception as e: | ... | ... |
| ... | ... | @@ -121,13 +121,16 @@ class Api(ApiTemplate): |
| 121 | 121 | |
| 122 | 122 | pg_ds :DataSource= PGUtil.open_pg_data_source(1,DES.decode(database.sqlalchemy_uri)) |
| 123 | 123 | |
| 124 | + | |
| 125 | + | |
| 126 | + | |
| 124 | 127 | # 创建抽稀过程 |
| 125 | 128 | options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name,pg_session)), |
| 126 | 129 | "PRECISION=NO"] |
| 127 | 130 | |
| 128 | 131 | layer = pg_ds.GetLayerByName(table.name) |
| 129 | 132 | |
| 130 | - vacuate_process:VacuateProcess = VacuateProcess(layer, table.name, options,database.sqlalchemy_uri,grids) | |
| 133 | + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database.sqlalchemy_uri,grids) | |
| 131 | 134 | |
| 132 | 135 | |
| 133 | 136 | for feature in layer: |
| ... | ... | @@ -233,7 +236,7 @@ class VacuateProcess: |
| 233 | 236 | this_gridsize=[] |
| 234 | 237 | |
| 235 | 238 | |
| 236 | - def __init__(self,layer:Layer,new_layer_name, options,sqlalchemy_uri,grids): | |
| 239 | + def __init__(self,layer:Layer,table_guid, options,sqlalchemy_uri,grids): | |
| 237 | 240 | |
| 238 | 241 | #是空间图层才初始化 |
| 239 | 242 | if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: |
| ... | ... | @@ -254,7 +257,9 @@ class VacuateProcess: |
| 254 | 257 | |
| 255 | 258 | # 创建抽稀ds |
| 256 | 259 | for l in range(self.max_level): |
| 257 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
| 260 | + | |
| 261 | + pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.SQLALCHEMY_DATABASE_URI) | |
| 262 | + # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
| 258 | 263 | pg_ds_l.StartTransaction() |
| 259 | 264 | self.pg_ds_dict[l] = pg_ds_l |
| 260 | 265 | |
| ... | ... | @@ -271,8 +276,8 @@ class VacuateProcess: |
| 271 | 276 | |
| 272 | 277 | if this_grid_len<1: |
| 273 | 278 | grid_name = str(this_grid_len).split(".")[-1] |
| 274 | - if this_grid_len.__eq__(0.000075): | |
| 275 | - grid_name = "000075" | |
| 279 | + if this_grid_len.__eq__(0.00008): | |
| 280 | + grid_name = "00008" | |
| 276 | 281 | else: |
| 277 | 282 | grid_name = str(int(this_grid_len)) |
| 278 | 283 | |
| ... | ... | @@ -284,8 +289,10 @@ class VacuateProcess: |
| 284 | 289 | |
| 285 | 290 | lev = this_grid_szie.index(this_grid_len) |
| 286 | 291 | print("{}:{}".format(grid_name, lev)) |
| 287 | - vl = pg.CreateLayer("{}_vacuate_{}_{}".format(new_layer_name, lev, grid_name), layer.GetSpatialRef(), | |
| 288 | - ogr.wkbUnknown, options) | |
| 292 | + | |
| 293 | + | |
| 294 | + v_ln = "z{}_vacuate_{}_{}".format(table_guid,lev, grid_name) | |
| 295 | + vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | |
| 289 | 296 | # 抽稀表不需要属性 |
| 290 | 297 | # vl.CreateFields(schema) |
| 291 | 298 | self.vacuate_layers[l] = vl |
| ... | ... | @@ -308,6 +315,13 @@ class VacuateProcess: |
| 308 | 315 | lat_extent = extent[3]-extent[2] |
| 309 | 316 | |
| 310 | 317 | this_grid_len =self.vacuate_layers_gridsize[level] |
| 318 | + #超大的直接加入 | |
| 319 | + # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | |
| 320 | + # vacuate_layer: Layer = self.vacuate_layers.get(level) | |
| 321 | + # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
| 322 | + # feat.SetGeometry(g) | |
| 323 | + # vacuate_layer.CreateFeature(feat) | |
| 324 | + # else: | |
| 311 | 325 | |
| 312 | 326 | row = int((center.GetY() - self.extent[2]) / this_grid_len) |
| 313 | 327 | col = int((center.GetX() - self.extent[0]) / this_grid_len) |
| ... | ... | @@ -325,7 +339,14 @@ class VacuateProcess: |
| 325 | 339 | else: |
| 326 | 340 | feat.SetGeometry(g) |
| 327 | 341 | vacuate_layer.CreateFeature(feat) |
| 328 | - self.fill_dict[key] += 1 | |
| 342 | + self.fill_dict[key] += 1 | |
| 343 | + #超大的还有机会 | |
| 344 | + elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | |
| 345 | + vacuate_layer: Layer = self.vacuate_layers.get(level) | |
| 346 | + feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
| 347 | + feat.SetGeometry(g) | |
| 348 | + vacuate_layer.CreateFeature(feat) | |
| 349 | + self.fill_dict[key] += 1 | |
| 329 | 350 | |
| 330 | 351 | def end(self): |
| 331 | 352 | for pg in self.pg_ds_dict.values(): | ... | ... |
| ... | ... | @@ -43,10 +43,6 @@ class Api(ApiTemplate): |
| 43 | 43 | layer = pg_ds.GetLayerByName(table.name) |
| 44 | 44 | |
| 45 | 45 | # 判断用户权限 |
| 46 | - user, pw, host, port, database = PGUtil.get_info_from_sqlachemy_uri(DES.decode(table.relate_database.sqlalchemy_uri)) | |
| 47 | - query_role:Layer = pg_ds.ExecuteSQL("SELECT rolcreatedb FROM pg_roles WHERE rolname='{}'".format(user)) | |
| 48 | - if not query_role.GetFeature(0).GetField("rolcreatedb"): | |
| 49 | - raise Exception("用户不具备建表权限!") | |
| 50 | 46 | |
| 51 | 47 | if not layer: |
| 52 | 48 | raise Exception("图层不存在!") | ... | ... |
app/util/component/CopyData2.py
0 → 100644
| 1 | +# coding=utf-8 | |
| 2 | +#author: 4N | |
| 3 | +#createtime: 2021/6/11 | |
| 4 | +#email: nheweijun@sina.com | |
| 5 | +import copy | |
| 6 | +from osgeo.ogr import * | |
| 7 | +from osgeo import gdal,ogr | |
| 8 | +import uuid | |
| 9 | +import time | |
| 10 | +import os | |
| 11 | +import json | |
| 12 | + | |
| 13 | + | |
| 14 | +def get_info_from_sqlachemy_uri(uri): | |
| 15 | + parts = uri.split(":") | |
| 16 | + user = parts[1][2:] | |
| 17 | + | |
| 18 | + password_list = parts[2].split("@") | |
| 19 | + if password_list.__len__() > 2: | |
| 20 | + password = "@".join(password_list[:-1]) | |
| 21 | + else: | |
| 22 | + password = parts[2].split("@")[0] | |
| 23 | + host = parts[2].split("@")[-1] | |
| 24 | + port = parts[3].split("/")[0] | |
| 25 | + database = parts[3].split("/")[1] | |
| 26 | + | |
| 27 | + return user, password, host, port, database | |
| 28 | + | |
| 29 | +def open_pg_data_source(iswrite, uri): | |
| 30 | + """ | |
| 31 | + # 获取PostGIS数据源 | |
| 32 | + :return: | |
| 33 | + """ | |
| 34 | + db_conn_tuple = get_info_from_sqlachemy_uri(uri) | |
| 35 | + fn = "PG: user=%s password=%s host=%s port=%s dbname=%s " % db_conn_tuple | |
| 36 | + driver = ogr.GetDriverByName("PostgreSQL") | |
| 37 | + if driver is None: | |
| 38 | + raise Exception("打开PostgreSQL驱动失败,可能是当前GDAL未支持PostgreSQL驱动!") | |
| 39 | + ds = driver.Open(fn, iswrite) | |
| 40 | + if ds is None: | |
| 41 | + raise Exception("打开数据源失败!") | |
| 42 | + return ds | |
| 43 | + | |
| 44 | + | |
| 45 | +def move_digui(coords,offx,offy): | |
| 46 | + if isinstance(coords[0],list): | |
| 47 | + for coord in coords: | |
| 48 | + move_digui(coord, offx, offy) | |
| 49 | + else : | |
| 50 | + coords[0] += offx | |
| 51 | + coords[1] += offy | |
| 52 | + | |
| 53 | +def move(geo,offx,offy): | |
| 54 | + | |
| 55 | + | |
| 56 | + geojson_str = geo.ExportToJson() | |
| 57 | + geojson = json.loads(geojson_str) | |
| 58 | + | |
| 59 | + coords = geojson["coordinates"] | |
| 60 | + move_digui(coords, offx, offy) | |
| 61 | + og = ogr.CreateGeometryFromJson(json.dumps(geojson)) | |
| 62 | + | |
| 63 | + return og | |
| 64 | + | |
| 65 | + | |
| 66 | +def copydata(bound,input,output): | |
| 67 | + work_dir = os.path.dirname(os.path.abspath(__file__)) | |
| 68 | + | |
| 69 | + shp_driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | |
| 70 | + bound_ds : DataSource = shp_driver.Open(bound, 0) | |
| 71 | + bound_layer :Layer = bound_ds.GetLayer(0) | |
| 72 | + | |
| 73 | + bf = bound_layer.GetNextFeature() | |
| 74 | + bound_geom = bf.GetGeometryRef() | |
| 75 | + | |
| 76 | + | |
| 77 | + | |
| 78 | + gdb_driver: Driver = ogr.GetDriverByName("FileGDB") | |
| 79 | + # gdb_ds: DataSource = gdb_driver.Open(gdb, 0) | |
| 80 | + # layer: Layer = gdb_ds.GetLayerByName(layer_name) | |
| 81 | + | |
| 82 | + input_ds: DataSource = shp_driver.Open(input, 0) | |
| 83 | + layer: Layer = input_ds.GetLayer(0) | |
| 84 | + | |
| 85 | + if not layer: | |
| 86 | + raise Exception("打开数据失败!") | |
| 87 | + | |
| 88 | + | |
| 89 | + output_ds: DataSource = gdb_driver.CreateDataSource(output) | |
| 90 | + | |
| 91 | + | |
| 92 | + gdb_layer:Layer = output_ds.CreateLayer(layer.GetName(),layer.GetSpatialRef(),layer.GetGeomType()) | |
| 93 | + | |
| 94 | + gdb_layer.CreateFields(layer.schema) | |
| 95 | + | |
| 96 | + | |
| 97 | + | |
| 98 | + | |
| 99 | + # | |
| 100 | + bound_exent = bound_layer.GetExtent() | |
| 101 | + | |
| 102 | + layer_extent = layer.GetExtent() | |
| 103 | + | |
| 104 | + xxrange=layer_extent[1]-layer_extent[0] | |
| 105 | + yrange = layer_extent[3]-layer_extent[2] | |
| 106 | + | |
| 107 | + | |
| 108 | + left = (int((layer_extent[0] - bound_exent[0]) / xxrange) + 1) * (-1) | |
| 109 | + right = int((bound_exent[1] - layer_extent[1]) / xxrange) + 1 | |
| 110 | + up = int((bound_exent[3] - layer_extent[3]) / yrange) + 1 | |
| 111 | + down = (int((layer_extent[2] - bound_exent[2]) / yrange) + 1) * (-1) | |
| 112 | + | |
| 113 | + | |
| 114 | + begin = time.time() | |
| 115 | + count=1 | |
| 116 | + | |
| 117 | + | |
| 118 | + for f in layer: | |
| 119 | + | |
| 120 | + new_f = copy.copy(f) | |
| 121 | + for xt in range(left,right+1,1): | |
| 122 | + for yt in range(down,up+1,1): | |
| 123 | + out_g = move(f.GetGeometryRef(),xxrange*xt,yrange*yt) | |
| 124 | + | |
| 125 | + if out_g.Intersect(bound_geom): | |
| 126 | + out_g = out_g.Intersection(bound_geom) | |
| 127 | + new_f.SetGeometry(out_g) | |
| 128 | + new_f.SetFID(count) | |
| 129 | + gdb_layer.CreateFeature(new_f) | |
| 130 | + count += 1 | |
| 131 | + if count % 10000 == 0: | |
| 132 | + # print(count) | |
| 133 | + with open(os.path.join(work_dir, "copy.txt"), "w") as fi: | |
| 134 | + fi.write("已完成{}".format(count)) | |
| 135 | + | |
| 136 | + print(time.time()-begin) | |
| 137 | + | |
| 138 | + input_ds.Destroy() | |
| 139 | + | |
| 140 | +def envelop_2_polygon(env): | |
| 141 | + ring = ogr.Geometry(ogr.wkbLinearRing) | |
| 142 | + ring.AddPoint(env[0], env[2]) | |
| 143 | + ring.AddPoint(env[0], env[3]) | |
| 144 | + ring.AddPoint(env[1], env[3]) | |
| 145 | + ring.AddPoint(env[1], env[2]) | |
| 146 | + ring.AddPoint(env[0], env[2]) | |
| 147 | + # Create polygon | |
| 148 | + poly = ogr.Geometry(ogr.wkbPolygon) | |
| 149 | + poly.AddGeometry(ring) | |
| 150 | + return poly | |
| 151 | + | |
| 152 | +def clip(): | |
| 153 | + | |
| 154 | + input = r"E:\Data\copy\x.shp" | |
| 155 | + output = r"E:\Data\copy\xianorigin.shp" | |
| 156 | + gdal.SetConfigOption("SHAPE_ENCODING", "UTF-8") | |
| 157 | + shp_driver: Driver = ogr.GetDriverByName("ESRI Shapefile") | |
| 158 | + | |
| 159 | + input_ds: DataSource = shp_driver.Open(input, 0) | |
| 160 | + layer: Layer = input_ds.GetLayer(0) | |
| 161 | + | |
| 162 | + bound_geo = envelop_2_polygon([112.84,112.952,22.914,22.985]) | |
| 163 | + | |
| 164 | + if not layer: | |
| 165 | + raise Exception("打开数据失败!") | |
| 166 | + | |
| 167 | + output_ds: DataSource = shp_driver.CreateDataSource(output) | |
| 168 | + | |
| 169 | + out_layer: Layer = output_ds.CreateLayer(layer.GetName(), layer.GetSpatialRef(), layer.GetGeomType()) | |
| 170 | + out_layer.CreateFields(layer.schema) | |
| 171 | + for f in layer: | |
| 172 | + new_f = copy.copy(f) | |
| 173 | + out_g = f.GetGeometryRef() | |
| 174 | + if out_g.Intersect(bound_geo): | |
| 175 | + out_g = out_g.Intersection(bound_geo) | |
| 176 | + new_f.SetGeometry(out_g) | |
| 177 | + # new_f.UnsetField("OBJECTID") | |
| 178 | + out_layer.CreateFeature(new_f) | |
| 179 | + | |
| 180 | + output_ds.Destroy() | |
| 181 | + | |
| 182 | + | |
| 183 | + | |
| 184 | +if __name__ == '__main__': | |
| 185 | + # bound_shp = "E:\Data\广东边界\gdsample.shp" | |
| 186 | + # input = r"E:\Data\copy\origin.shp" | |
| 187 | + # output = r"E:\Data\copy\re.shp" | |
| 188 | + # copydata(bound_shp,input,output) | |
| 189 | + | |
| 190 | + # bound_shp = "/root/CopyData/gdsample.shp" | |
| 191 | + # input = r"/root/CopyData/origin.shp" | |
| 192 | + # output = r"/root/CopyData/re.gdb" | |
| 193 | + # copydata(bound_shp,input,output) | |
| 194 | + clip() | |
| 195 | + | |
| 196 | + | |
| 197 | + | |
| 198 | + | |
| 199 | + | |
| 200 | + | |
| 201 | + | |
| 202 | + | |
| 203 | + | |
| 204 | + | |
| 205 | + | ... | ... |
| ... | ... | @@ -159,7 +159,7 @@ class EntryDataVacuate: |
| 159 | 159 | new_layer_name = None |
| 160 | 160 | vacuate_process= None |
| 161 | 161 | success = True |
| 162 | - | |
| 162 | + table_guid = uuid.uuid1().__str__() | |
| 163 | 163 | try: |
| 164 | 164 | # 图层设置 |
| 165 | 165 | parameter = this_task.parameter |
| ... | ... | @@ -200,7 +200,7 @@ class EntryDataVacuate: |
| 200 | 200 | pg_layer.CreateFields(schema) |
| 201 | 201 | |
| 202 | 202 | #创建抽稀过程 |
| 203 | - vacuate_process = VacuateProcess(layer,new_layer_name,options,this_task.database.sqlalchemy_uri) | |
| 203 | + vacuate_process = VacuateProcess(layer,table_guid,options) | |
| 204 | 204 | |
| 205 | 205 | |
| 206 | 206 | |
| ... | ... | @@ -235,7 +235,8 @@ class EntryDataVacuate: |
| 235 | 235 | # 注册图层信息 |
| 236 | 236 | # 是否抽吸过 |
| 237 | 237 | is_vacuate = 1 if vacuate_process.max_level>0 else 0 |
| 238 | - table_guid = this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate) | |
| 238 | + | |
| 239 | + this_task.register_table(pg_layer,new_layer_name,overwrite,parameter.get("creator"),is_vacuate,table_guid) | |
| 239 | 240 | |
| 240 | 241 | # 注册抽稀表 |
| 241 | 242 | this_task.register_table_vacuate(table_guid,vacuate_process.vacuate_layers) |
| ... | ... | @@ -295,7 +296,7 @@ class ThisTask: |
| 295 | 296 | self.process_session.add(task_process) |
| 296 | 297 | self.process_session.commit() |
| 297 | 298 | |
| 298 | - def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate): | |
| 299 | + def register_table(self, layer: Layer, new_layer_name, overwrite, creator,is_vacuate,table_guid): | |
| 299 | 300 | ''' |
| 300 | 301 | 注册表 |
| 301 | 302 | :param layer: 图层 |
| ... | ... | @@ -321,7 +322,7 @@ class ThisTask: |
| 321 | 322 | geom_type = GeometryAdapter.get_geometry_type(layer) |
| 322 | 323 | |
| 323 | 324 | extent = "{},{},{},{}".format(ext[0], ext[1], ext[2], ext[3]) |
| 324 | - table_guid = uuid.uuid1().__str__() | |
| 325 | + | |
| 325 | 326 | table = Table(guid=table_guid, |
| 326 | 327 | database_guid=self.database.guid, |
| 327 | 328 | creator=creator, |
| ... | ... | @@ -419,7 +420,7 @@ class VacuateProcess: |
| 419 | 420 | this_gridsize=[] |
| 420 | 421 | |
| 421 | 422 | |
| 422 | - def __init__(self,layer:Layer,new_layer_name, options,sqlalchemy_uri): | |
| 423 | + def __init__(self,layer:Layer,table_guid, options): | |
| 423 | 424 | |
| 424 | 425 | #是空间图层才初始化 |
| 425 | 426 | if layer.GetExtent()[0] > 0 or layer.GetExtent()[0] < 0: |
| ... | ... | @@ -466,6 +467,7 @@ class VacuateProcess: |
| 466 | 467 | self.t_grid_size=self.project_gridsize |
| 467 | 468 | else: |
| 468 | 469 | self.t_grid_size = self.lonlat_gridsize |
| 470 | + | |
| 469 | 471 | for grid_size in self.t_grid_size: |
| 470 | 472 | # 最少抽稀个数 |
| 471 | 473 | if lc > self.least_vacuate_count: |
| ... | ... | @@ -482,7 +484,8 @@ class VacuateProcess: |
| 482 | 484 | |
| 483 | 485 | # 创建抽稀ds |
| 484 | 486 | for l in range(self.max_level): |
| 485 | - pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
| 487 | + # pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, DES.decode(sqlalchemy_uri)) | |
| 488 | + pg_ds_l: DataSource = PGUtil.open_pg_data_source(1, configure.SQLALCHEMY_DATABASE_URI) | |
| 486 | 489 | pg_ds_l.StartTransaction() |
| 487 | 490 | self.pg_ds_dict[l] = pg_ds_l |
| 488 | 491 | |
| ... | ... | @@ -500,16 +503,19 @@ class VacuateProcess: |
| 500 | 503 | grid_name = str(this_grid_len) |
| 501 | 504 | if this_grid_len<1: |
| 502 | 505 | grid_name = str(this_grid_len).split(".")[-1] |
| 503 | - if this_grid_len.__eq__(0.000075): | |
| 504 | - grid_name = "000075" | |
| 506 | + if this_grid_len.__eq__(0.00008): | |
| 507 | + grid_name = "00008" | |
| 505 | 508 | |
| 506 | 509 | # 抽稀图层是点面混合的 |
| 507 | 510 | # 抽稀表有固定的命名规则 |
| 508 | 511 | # 抽稀表一定要覆盖 |
| 509 | 512 | |
| 510 | 513 | |
| 511 | - vl = pg.CreateLayer("{}_vacuate_{}_{}".format(new_layer_name, self.t_grid_size.index(this_grid_len), grid_name), layer.GetSpatialRef(), | |
| 512 | - ogr.wkbUnknown, options) | |
| 514 | + print("{}:{}".format(self.t_grid_size.index(this_grid_len),this_grid_len)) | |
| 515 | + | |
| 516 | + | |
| 517 | + v_ln = "z{}_vacuate_{}_{}".format(table_guid, self.t_grid_size.index(this_grid_len), grid_name) | |
| 518 | + vl = pg.CreateLayer(v_ln, layer.GetSpatialRef(),ogr.wkbUnknown, options) | |
| 513 | 519 | # 抽稀表不需要属性 |
| 514 | 520 | # vl.CreateFields(schema) |
| 515 | 521 | self.vacuate_layers[l] = vl |
| ... | ... | @@ -532,6 +538,13 @@ class VacuateProcess: |
| 532 | 538 | lat_extent = extent[3]-extent[2] |
| 533 | 539 | |
| 534 | 540 | this_grid_len =self.vacuate_layers_gridsize[level] |
| 541 | + #超大的直接加入 | |
| 542 | + # if long_extent > 10*this_grid_len or lat_extent >10*this_grid_len: | |
| 543 | + # vacuate_layer: Layer = self.vacuate_layers.get(level) | |
| 544 | + # feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
| 545 | + # feat.SetGeometry(g) | |
| 546 | + # vacuate_layer.CreateFeature(feat) | |
| 547 | + # else: | |
| 535 | 548 | |
| 536 | 549 | row = int((center.GetY() - self.extent[2]) / this_grid_len) |
| 537 | 550 | col = int((center.GetX() - self.extent[0]) / this_grid_len) |
| ... | ... | @@ -549,7 +562,14 @@ class VacuateProcess: |
| 549 | 562 | else: |
| 550 | 563 | feat.SetGeometry(g) |
| 551 | 564 | vacuate_layer.CreateFeature(feat) |
| 552 | - self.fill_dict[key] += 1 | |
| 565 | + self.fill_dict[key] += 1 | |
| 566 | + #超大的还有机会 | |
| 567 | + elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5: | |
| 568 | + vacuate_layer: Layer = self.vacuate_layers.get(level) | |
| 569 | + feat = ogr.Feature(vacuate_layer.GetLayerDefn()) | |
| 570 | + feat.SetGeometry(g) | |
| 571 | + vacuate_layer.CreateFeature(feat) | |
| 572 | + self.fill_dict[key] += 1 | |
| 553 | 573 | |
| 554 | 574 | def end(self): |
| 555 | 575 | for pg in self.pg_ds_dict.values(): | ... | ... |
app/util/component/copy.txt
0 → 100644
| 1 | +メムヘノ20000 | |
| \ No newline at end of file | ... | ... |
| ... | ... | @@ -355,6 +355,9 @@ IncludeOptional conf.d/*.conf |
| 355 | 355 | LoadModule wsgi_module "/usr/lib64/httpd/modules/mod_wsgi-py37.cpython-37m-x86_64-linux-gnu.so" |
| 356 | 356 | "/var/gdal" |
| 357 | 357 | |
| 358 | + dmapmanager processes=4 threads=16 display-name=%{GROUP} | |
| 359 | + dmapmanager | |
| 360 | + | |
| 358 | 361 | / /usr/src/app/run.wsgi |
| 359 | 362 | <Directory /usr/> |
| 360 | 363 | Require all granted | ... | ... |
| ... | ... | @@ -19,15 +19,5 @@ import json |
| 19 | 19 | # print(sys.getsizeof(json.dumps(pixel_array.tolist()))) |
| 20 | 20 | # print(sys.getsizeof((content))) |
| 21 | 21 | |
| 22 | -def c(dd): | |
| 23 | - return 1 | |
| 24 | - | |
| 25 | -dd = numpy.zeros((2000, 2000), dtype=int) + 2 | |
| 26 | - | |
| 27 | - | |
| 28 | -dd[range(0, 5), 1]=1 | |
| 29 | - | |
| 30 | - | |
| 31 | - | |
| 32 | - | |
| 33 | -print(dd) | |
| \ No newline at end of file | ||
| 22 | +for x in range(-3,3,1): | |
| 23 | + print(x) | |
| \ No newline at end of file | ... | ... |
请
注册
或
登录
后发表评论