HWJ 何伟俊 / DMapManager · 提交

提交 237b014b5792f920fe89457466a5f4e399857600

作者 nheweijun
1 个父辈 00fdfd17

2021.11.25

内嵌 并排对比
正在显示 45 个修改的文件 包含 752 行增加 和 193 行删除
... ... @@ -78,7 +78,7 @@ def create_app():
78 78 db.create_all(app=app)
79 79
80 80 # 日志
81   - logging.basicConfig(level=logging.INFO)
  81 + logging.basicConfig(level=configure.log_level)
82 82 log_file = os.path.join(os.path.dirname(os.path.dirname(
83 83 os.path.realpath(__file__))), "logs", "log.txt")
84 84 handler = logging.FileHandler(
... ...
... ... @@ -37,10 +37,12 @@ class Api(ApiTemplate):
37 37 elif self.para.get("data_path").endswith("shp"):
38 38 data_path=self.para.get("data_path")
39 39 encoding_cpg_path = data_path.split(".shp")[0]+".cpg"
40   - with open(encoding_cpg_path) as fd:
41   - encoding_cpg = fd.readline().strip()
42 40 if not os.path.exists(encoding_cpg_path):
43 41 encoding_cpg=None
  42 + else:
  43 + with open(encoding_cpg_path) as fd:
  44 + encoding_cpg = fd.readline().strip()
  45 +
44 46 spatial_files.append((data_path, encoding_cpg))
45 47 elif self.para.get("data_path").endswith("gdb"):
46 48 data_path=self.para.get("data_path")
... ...
... ... @@ -22,6 +22,7 @@ from . import table_vacuate_one
22 22 from . import table_vacuate_info
23 23 from . import table_vacuate_ref
24 24 from . import table_vacuate_delete
  25 +from . import field_value
25 26 class DataManager(BlueprintApi):
26 27
27 28 bp = Blueprint("DataManager", __name__, url_prefix="/API/Manager")
... ... @@ -45,6 +46,15 @@ class DataManager(BlueprintApi):
45 46 return field_list.Api().result
46 47
47 48 @staticmethod
  49 + @bp.route('/FieldValue', methods=['POST'])
  50 + @swag_from(field_value.Api.api_doc)
  51 + def field_value():
  52 + """
  53 + 属性值
  54 + """
  55 + return field_value.Api().result
  56 +
  57 + @staticmethod
48 58 @bp.route('/TableList', methods=['POST'])
49 59 @swag_from(table_list.Api.api_doc)
50 60 def table_list():
... ...
... ... @@ -18,9 +18,18 @@ class Api(ApiTemplate):
18 18 res={}
19 19
20 20 column_guid = self.para.get("column_guid")
  21 + update_dict = {}
21 22 try:
22   - Columns.query.filter_by(guid = column_guid).update({"alias":self.para.get("column_alias"),"update_time":datetime.datetime.now()})
  23 + if self.para.get("column_alias"):
  24 + update_dict["alias"] = self.para.get("column_alias")
  25 + if self.para.get("is_for_partition") is not None:
  26 + update_dict["is_for_partition"] = int(self.para.get("is_for_partition"))
  27 + if update_dict:
  28 + update_dict["update_time"] = datetime.datetime.now()
  29 + Columns.query.filter_by(guid = column_guid).update(update_dict)
  30 +
23 31 db.session.commit()
  32 +
24 33 res["result"] = True
25 34 res["msg"] = "属性别名修改成功！"
26 35
... ... @@ -29,6 +38,9 @@ class Api(ApiTemplate):
29 38
30 39 return res
31 40
  41 +
  42 +
  43 +
32 44 api_doc={
33 45 "tags":["管理接口"],
34 46 "parameters":[
... ... @@ -39,7 +51,11 @@ class Api(ApiTemplate):
39 51 {"name": "column_alias",
40 52 "in": "formData",
41 53 "type": "string",
42   - "description": "属性别名","required":"true"},
  54 + "description": "属性别名"},
  55 + {"name": "is_for_partition",
  56 + "in": "formData",
  57 + "type": "int",
  58 + "description": "是否用于分层分级", "enum": [0, 1]},
43 59
44 60 ],
45 61 "responses":{
... ...
... ... @@ -18,10 +18,13 @@ class Api(ApiTemplate):
18 18
19 19 try:
20 20 table_guid = self.para.get("guid")
  21 + is_for_partition = self.para.get("is_for_partition")
21 22 table:Table = Table.query.filter_by(guid=table_guid).one_or_none()
22 23 if table:
23 24 res["result"]=True
24 25 res["data"] = ModelVisitor.objects_to_jsonarray(table.relate_columns)
  26 + if is_for_partition:
  27 + res["data"] = [data for data in res["data"] if data["is_for_partition"] == int(is_for_partition)]
25 28 else:
26 29 res["result"]=False
27 30 res["msg"] = "数据表不存在！"
... ... @@ -38,6 +41,10 @@ class Api(ApiTemplate):
38 41 "in": "formData",
39 42 "type": "string",
40 43 "description": "表guid"},
  44 + {"name": "is_for_partition",
  45 + "in": "formData",
  46 + "type": "int",
  47 + "description": "是否用于分层分级", "enum": [0, 1]},
41 48
42 49 ],
43 50 "responses":{
... ...
  1 +#author: 4N
  2 +#createtime: 2021/1/27
  3 +#email: nheweijun@sina.com
  4 +
  5 +
  6 +import traceback
  7 +from ..models import Table,Columns,DES
  8 +from app.util.component.ApiTemplate import ApiTemplate
  9 +from app.util.component.PGUtil import PGUtil
  10 +from app.util.component.ModelVisitor import ModelVisitor
  11 +from sqlalchemy.orm import Session
  12 +
  13 +
  14 +class Api(ApiTemplate):
  15 + api_name = "属性列表"
  16 + def process(self):
  17 +
  18 + #返回结果
  19 + res={}
  20 + system_session = None
  21 +
  22 + try:
  23 + table_guid = self.para.get("table_guid")
  24 + column_guid = self.para.get("column_guid")
  25 +
  26 + page_index = int(self.para.get("page_index", "0"))
  27 + page_size = int(self.para.get("page_size", "10"))
  28 +
  29 +
  30 +
  31 + table:Table = Table.query.filter_by(guid=table_guid).one_or_none()
  32 + column:Columns = Columns.query.filter_by(guid=column_guid).one_or_none()
  33 + system_session: Session = PGUtil.get_db_session(DES.decode(table.relate_database.sqlalchemy_uri))
  34 +
  35 + if self.para.get("key"):
  36 + query = '''where "{}" like '%{}%' '''.format(column.name,self.para.get("key"))
  37 + else :
  38 + query = ""
  39 +
  40 +
  41 +
  42 + query_sql = 'select distinct "{}" from public."{}" {} order by "{}" limit {} offset {}'.format(
  43 + column.name,table.name,query, column.name,page_size,page_index * page_size
  44 + )
  45 +
  46 +
  47 + query_res = system_session.execute(query_sql).fetchall()
  48 +
  49 + query_count = system_session.execute('select count(distinct {}) from public."{}" {} '.format(
  50 + column.name,table.name,query
  51 + )).fetchone()
  52 +
  53 + res["data"] = {}
  54 + res["data"]["count"] = query_count[0]
  55 + res["data"]["list"] = [dat[0] for dat in query_res]
  56 + res["result"] = True
  57 +
  58 + except Exception as e:
  59 + print(traceback.format_exc())
  60 + raise e
  61 + finally:
  62 + try:
  63 + system_session.close()
  64 + except:
  65 + pass
  66 + return res
  67 +
  68 + api_doc={
  69 + "tags":["管理接口"],
  70 + "parameters":[
  71 + {"name": "table_guid",
  72 + "in": "formData",
  73 + "type": "string",
  74 + "description": "表guid"},
  75 + {"name": "column_guid",
  76 + "in": "formData",
  77 + "type": "string",
  78 + "description": "表guid"},
  79 + {"name": "page_index",
  80 + "in": "formData",
  81 + "type": "int",
  82 + "description": "页"},
  83 + {"name": "page_size",
  84 + "in": "formData",
  85 + "type": "int",
  86 + "description": "页大小"},
  87 + {"name": "key",
  88 + "in": "formData",
  89 + "type": "string",
  90 + "description": "key检索关键字"},
  91 +
  92 + ],
  93 + "responses":{
  94 + 200:{
  95 + "schema":{
  96 + "properties":{
  97 + }
  98 + }
  99 + }
  100 + }
  101 + }
\ No newline at end of file
... ...
... ... @@ -26,7 +26,7 @@ class Api(ApiTemplate):
26 26 description = self.para.get("description")
27 27 name = self.para.get("name")
28 28 columns_edit = self.para.get("columns_edit")
29   -
  29 + is_for_partition = self.para.get("is_for_partition")
30 30
31 31 table = Table.query.filter_by(guid=table_guid)
32 32 if not table.one_or_none():
... ... @@ -54,6 +54,8 @@ class Api(ApiTemplate):
54 54
55 55 if self.para.__contains__("alias"):
56 56 table.update({"alias":table_alias})
  57 + if is_for_partition is not None:
  58 + table.update({"is_for_partition": int(is_for_partition)})
57 59
58 60 if self.para.__contains__("description"):
59 61 table.update({"description":description})
... ... @@ -61,7 +63,13 @@ class Api(ApiTemplate):
61 63 if columns_edit:
62 64 columns_edit_list = json.loads(columns_edit)
63 65 for ce in columns_edit_list:
64   - Columns.query.filter_by(guid=ce["guid"]).update({"alias": ce.get("alias")})
  66 + columns_update_dict = {}
  67 + if ce.get("alias"):
  68 + columns_update_dict["alias"] = ce.get("alias")
  69 + if ce.get("is_for_partition") is not None:
  70 + columns_update_dict["is_for_partition"] = int(ce.get("is_for_partition"))
  71 + if columns_update_dict:
  72 + Columns.query.filter_by(guid=ce["guid"]).update(columns_update_dict)
65 73
66 74 if name:
67 75 sys_session=None
... ... @@ -110,6 +118,10 @@ class Api(ApiTemplate):
110 118 "in": "formData",
111 119 "type": "string",
112 120 "description": "表名"},
  121 + {"name": "is_for_partition",
  122 + "in": "formData",
  123 + "type": "int",
  124 + "description": "是否用于分层分级", "enum": [0, 1]},
113 125 {"name": "description",
114 126 "in": "formData",
115 127 "type": "string",
... ...
... ... @@ -30,7 +30,7 @@ class Api(ApiTemplate):
30 30
31 31 start_time = self.para.get("start_time")
32 32 end_time = self.para.get("end_time")
33   -
  33 + is_for_partition = self.para.get("is_for_partition")
34 34
35 35
36 36 recursion = int(self.para.get("recursion","0"))
... ... @@ -46,6 +46,8 @@ class Api(ApiTemplate):
46 46
47 47 if database_guid:
48 48 tables = tables.filter_by(database_guid=database_guid)
  49 + if is_for_partition:
  50 + tables = tables.filter_by(is_for_partition=int(is_for_partition))
49 51 if catalog_guid:
50 52
51 53 if recursion.__eq__(0):
... ... @@ -80,10 +82,7 @@ class Api(ApiTemplate):
80 82 except Exception as e:
81 83 raise e
82 84 return res
83   -
84   -
85   -
86   -
  85 +
87 86
88 87 api_doc={
89 88 "tags":["管理接口"],
... ... @@ -108,6 +107,10 @@ class Api(ApiTemplate):
108 107 "in": "formData",
109 108 "type": "int",
110 109 "description": "点线面123","enum":[1,2,3]},
  110 + {"name": "is_for_partition",
  111 + "in": "formData",
  112 + "type": "int",
  113 + "description": "是否用于分层分级", "enum": [0, 1]},
111 114 {"name": "database_guid",
112 115 "in": "formData",
113 116 "type": "string",
... ...
... ... @@ -501,10 +501,10 @@ class Api(ApiTemplate):
501 501 "type": "string",
502 502 "description": "数据库guid",
503 503 "required":"true"},
504   - {"name": "user",
  504 + {"name": "creator",
505 505 "in": "formData",
506 506 "type": "string",
507   - "description": "用户"}
  507 + "description": "创建者"}
508 508
509 509 ],
510 510 "responses":{
... ...
... ... @@ -5,25 +5,30 @@
5 5
6 6 import datetime
7 7 import traceback
  8 +import multiprocessing
  9 +import uuid
  10 +import configure
  11 +
8 12 from ..models import Table, Database, Task,db,TableVacuate,Process,DES
9 13
10   -from sqlalchemy.engine import ResultProxy
11 14 from app.util.component.ApiTemplate import ApiTemplate
12   -
13 15 from app.util.component.StructuredPrint import StructurePrint
14 16 from app.util.component.PGUtil import PGUtil
15   -import multiprocessing
16   -import uuid
17   -import configure
18   -from osgeo.ogr import DataSource,Layer,Geometry
19   -from osgeo import ogr
20 17 from app.util.component.VacuateConf import VacuateConf
21 18 from app.util.component.GeometryAdapter import GeometryAdapter
22 19
23   -from .table_vacuate_ref import Api as RefApi
  20 +
  21 +from osgeo.ogr import DataSource,Layer,Geometry
  22 +from osgeo import ogr
  23 +
  24 +
  25 +from sqlalchemy.orm import Session
  26 +
24 27
25 28 class Api(ApiTemplate):
  29 +
26 30 api_name = "数据抽稀"
  31 +
27 32 def process(self):
28 33
29 34 res = {}
... ... @@ -59,21 +64,15 @@ class Api(ApiTemplate):
59 64 res["msg"] = "非空间表！"
60 65 return res
61 66
62   - # if table.is_vacuate==1:
63   - # res["state"] = -1
64   - # res["message"] = "已精化！"
65   - # return res
66   -
67   -
68 67
69 68 # 初始化task
70 69 task_guid = uuid.uuid1().__str__()
71 70 vacuate_process = multiprocessing.Process(target=self.task,args=(table,task_guid))
72 71 vacuate_process.start()
73 72
74   - ref_api = RefApi()
75   - ref_api.para["guid"] = table_guid
76   - ref_grids = ref_api.process()["data"]
  73 + # ref_api = RefApi()
  74 + # ref_api.para["guid"] = table_guid
  75 + # ref_grids = ref_api.process()["data"]
77 76
78 77 task = Task(guid=task_guid,
79 78 name="{}精化".format(table.name),
... ... @@ -85,7 +84,8 @@ class Api(ApiTemplate):
85 84 file_name=None,
86 85 database_guid=table.database_guid,
87 86 process="精化中",
88   - parameter=",".join([str(x) for x in ref_grids]))
  87 + # parameter=",".join([str(x) for x in ref_grids])
  88 + )
89 89
90 90 db.session.add(task)
91 91 db.session.commit()
... ... @@ -108,12 +108,12 @@ class Api(ApiTemplate):
108 108 vacuate_process = None
109 109 try:
110 110 sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
111   - sys_session.query(Table).filter_by(guid=table.guid).update(
112   - {"is_vacuate": 2, "update_time": datetime.datetime.now()})
113   - sys_session.commit()
  111 +
  112 + sys_session.query(Table).filter_by(guid=table.guid).update({"is_vacuate": 2, "update_time": datetime.datetime.now()})
114 113
115 114
116 115 database = sys_session.query(Database).filter_by(guid=table.database_guid).one_or_none()
  116 + database_sqlalchemy_uri = str(database.sqlalchemy_uri)
117 117 pg_session = PGUtil.get_db_session(DES.decode(database.sqlalchemy_uri))
118 118
119 119 pg_ds :DataSource= PGUtil.open_pg_data_source(0,DES.decode(database.sqlalchemy_uri))
... ... @@ -123,10 +123,10 @@ class Api(ApiTemplate):
123 123 tvs = sys_session.query(TableVacuate).filter_by(table_guid=table.guid).all()
124 124 for tv in tvs :
125 125 sys_session.delete(tv)
126   - # try:
127   - # pg_ds.DeleteLayer(tv.name)
128   - # except Exception as e :
129   - # StructurePrint().print("抽稀图层不存在！","warn")
  126 +
  127 + sys_session.commit()
  128 + #长时间的连接，导致后续的session超时，现在先断开
  129 + sys_session.close()
130 130
131 131 # 创建抽稀过程
132 132 options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name,pg_session)),
... ... @@ -134,7 +134,7 @@ class Api(ApiTemplate):
134 134
135 135 layer = pg_ds.GetLayerByName(table.name)
136 136
137   - vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database.sqlalchemy_uri)
  137 + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri)
138 138
139 139
140 140 for feature in layer:
... ... @@ -145,11 +145,14 @@ class Api(ApiTemplate):
145 145
146 146 vacuate_process.set_vacuate_count()
147 147
  148 + # 重连
  149 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
  150 +
148 151 #新增
149 152 if configure.VACUATE_DB_URI:
150 153 user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(configure.VACUATE_DB_URI)
151 154 else:
152   - user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(DES.decode(database.sqlalchemy_uri))
  155 + user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(DES.decode(database_sqlalchemy_uri))
153 156 connectstr = "hostaddr={} port={} dbname='{}' user='{}' password='{}'".format(host, port, datab, user,
154 157 passwd)
155 158 for l in range(vacuate_process.max_level):
... ... @@ -171,6 +174,8 @@ class Api(ApiTemplate):
171 174
172 175 except Exception as e:
173 176 try:
  177 + if not sys_session.is_active:
  178 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
174 179 sys_session.query(Task).filter_by(guid=task_guid).update({"state": -1,"update_time":datetime.datetime.now(),
175 180 "process": "精化失败"})
176 181 sys_session.query(Table).filter_by(guid=table.guid).update(
... ... @@ -189,16 +194,18 @@ class Api(ApiTemplate):
189 194 except Exception as ee:
190 195 print(traceback.format_exc())
191 196 finally:
192   - if vacuate_process:
193   - vacuate_process.end()
194   - if sys_session:
195   - sys_session.close()
196   - if pg_session:
197   - pg_session.close()
198   - if pg_ds:
199   - pg_ds.Destroy()
200   -
201   -
  197 + try:
  198 + if vacuate_process:
  199 + vacuate_process.end()
  200 + if sys_session:
  201 + sys_session.close()
  202 + if pg_session:
  203 + pg_session.close()
  204 + if pg_ds:
  205 + pg_ds.Destroy()
  206 +
  207 + except Exception as e:
  208 + StructurePrint.print(traceback.format_exc())
202 209 api_doc = {
203 210 "tags": ["管理接口"],
204 211 "parameters": [
... ... @@ -206,6 +213,10 @@ class Api(ApiTemplate):
206 213 "in": "formData",
207 214 "type": "string",
208 215 "description": "表guid", "required": "true"},
  216 + {"name": "creator",
  217 + "in": "formData",
  218 + "type": "string",
  219 + "description": "创建者"}
209 220 ],
210 221 "responses": {
211 222 200: {
... ... @@ -217,6 +228,16 @@ class Api(ApiTemplate):
217 228 }
218 229 }
219 230
  231 +class SysSession:
  232 + def __init__(self):
  233 + self.session : Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
  234 +
  235 + def close(self):
  236 + try:
  237 + self.session.close()
  238 + except:
  239 + pass
  240 +
220 241 class VacuateProcess:
221 242
222 243 max_level=0
... ... @@ -274,8 +295,7 @@ class VacuateProcess:
274 295
275 296 layer.SetSpatialFilter(None)
276 297 layer.ResetReading()
277   - # 固有疏密程度
278   - original_density=8
  298 +
279 299
280 300
281 301 # 额外一层
... ... @@ -295,7 +315,7 @@ class VacuateProcess:
295 315 if ((extent[1] - extent[0]) * (extent[3] - extent[2])) / (grid_size**2)>self.least_vacuate_count:
296 316 # 要素数量大于网格数量
297 317 # 要考虑图层的疏密程度，original_density*(100.0/fill_precent) 为疏密指数
298   - if lc * original_density * (100.0/fill_precent)>((extent[1] - extent[0])*(extent[3] - extent[2]))/(grid_size**2) :
  318 + if lc * VacuateConf.original_density * (100.0/fill_precent)>((extent[1] - extent[0])*(extent[3] - extent[2]))/(grid_size**2) :
299 319 print(grid_size)
300 320 self.this_gridsize.append(grid_size)
301 321 self.max_level += 1
... ... @@ -400,7 +420,8 @@ class VacuateProcess:
400 420 vacuate_layer.CreateFeature(feat)
401 421 self.fill_dict[key] += 1
402 422 #超大的还有机会
403   - elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:
  423 + # elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:
  424 + elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len:
404 425 vacuate_layer: Layer = self.vacuate_layers.get(level)
405 426 feat = ogr.Feature(vacuate_layer.GetLayerDefn())
406 427 feat.SetGeometry(g)
... ...
... ... @@ -113,16 +113,24 @@ class Api(ApiTemplate):
113 113 sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
114 114 sys_session.query(Table).filter_by(guid=table.guid).update(
115 115 {"is_vacuate": 2, "update_time": datetime.datetime.now()})
116   - sys_session.commit()
117   -
118 116
119 117 database = sys_session.query(Database).filter_by(guid=table.database_guid).one_or_none()
120   - pg_session = PGUtil.get_db_session(DES.decode(database.sqlalchemy_uri))
121 118
122   - pg_ds :DataSource= PGUtil.open_pg_data_source(1,DES.decode(database.sqlalchemy_uri))
  119 + database_sqlalchemy_uri = str(database.sqlalchemy_uri)
  120 +
  121 + pg_session = PGUtil.get_db_session(DES.decode(database_sqlalchemy_uri))
123 122
  123 + pg_ds :DataSource= PGUtil.open_pg_data_source(1,DES.decode(database_sqlalchemy_uri))
124 124
  125 + #删除原有数据
  126 + for grid in grids:
  127 + tvs = sys_session.query(TableVacuate).filter_by(pixel_distance=grid,table_guid=table.guid).all()
  128 + for tv in tvs :
  129 + sys_session.delete(tv)
125 130
  131 + sys_session.commit()
  132 + #长时间的连接，导致后续的session超时，现在先断开
  133 + sys_session.close()
126 134
127 135 # 创建抽稀过程
128 136 options = ["OVERWRITE=yes", "GEOMETRY_NAME={}".format(PGUtil.get_geo_column(table.name,pg_session)),
... ... @@ -130,7 +138,7 @@ class Api(ApiTemplate):
130 138
131 139 layer = pg_ds.GetLayerByName(table.name)
132 140
133   - vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database.sqlalchemy_uri,grids)
  141 + vacuate_process:VacuateProcess = VacuateProcess(layer, table.guid, options,database_sqlalchemy_uri,grids)
134 142
135 143
136 144 for feature in layer:
... ... @@ -141,18 +149,14 @@ class Api(ApiTemplate):
141 149
142 150 vacuate_process.set_vacuate_count()
143 151
144   - #删除原有数据
145   - for grid in grids:
146   - tvs = sys_session.query(TableVacuate).filter_by(pixel_distance=grid,table_guid=table.guid).all()
147   - for tv in tvs :
148   - sys_session.delete(tv)
149   -
  152 + #重新连接
  153 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
150 154
151 155 #新增
152 156 if configure.VACUATE_DB_URI:
153 157 user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(configure.VACUATE_DB_URI)
154 158 else:
155   - user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(DES.decode(database.sqlalchemy_uri))
  159 + user, passwd, host, port, datab = PGUtil.get_info_from_sqlachemy_uri(DES.decode(database_sqlalchemy_uri))
156 160 connectstr = "hostaddr={} port={} dbname='{}' user='{}' password='{}'".format(host, port, datab, user,
157 161 passwd)
158 162 for l in range(vacuate_process.max_level):
... ... @@ -175,6 +179,8 @@ class Api(ApiTemplate):
175 179
176 180 except Exception as e:
177 181 try:
  182 + if not sys_session.is_active:
  183 + sys_session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
178 184 sys_session.query(Task).filter_by(guid=task_guid).update({"state": -1,"update_time":datetime.datetime.now(),
179 185 "process": "精化失败"})
180 186 sys_session.query(Table).filter_by(guid=table.guid).update(
... ... @@ -215,7 +221,12 @@ class Api(ApiTemplate):
215 221 {"name": "grids",
216 222 "in": "formData",
217 223 "type": "string",
218   - "description": "需要抽稀的网格大小，以逗号相隔", "required": "true"}
  224 + "description": "需要抽稀的网格大小，以逗号相隔", "required": "true"},
  225 + {"name": "creator",
  226 + "in": "formData",
  227 + "type": "string",
  228 + "description": "创建者"}
  229 +
219 230 ],
220 231 "responses": {
221 232 200: {
... ... @@ -369,7 +380,8 @@ class VacuateProcess:
369 380 self.fill_dict[key] += 1
370 381
371 382 #超大的还有机会
372   - elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:
  383 + # elif (long_extent > 10*this_grid_len or lat_extent >10*this_grid_len) and self.fill_dict[key]<5:
  384 + elif long_extent > 10 * this_grid_len or lat_extent > 10 * this_grid_len:
373 385 vacuate_layer: Layer = self.vacuate_layers.get(level)
374 386 feat = ogr.Feature(vacuate_layer.GetLayerDefn())
375 387 feat.SetGeometry(g)
... ...
... ... @@ -55,7 +55,8 @@ class Table(db.Model):
55 55 creator = Column(Text)
56 56 #是否已抽稀
57 57 is_vacuate=Column(Integer,default=0)
58   -
  58 + #是否用于分层分级
  59 + is_for_partition = Column(Integer,default=0)
59 60 # 目录外键
60 61 catalog_guid = Column(String(256), ForeignKey('dmap_catalog.guid'))
61 62
... ... @@ -89,6 +90,8 @@ class Columns(db.Model):
89 90 alias = Column(String(256))
90 91 create_time = Column(DateTime)
91 92 update_time = Column(DateTime)
  93 + #是否用于分层分级
  94 + is_for_partition = Column(Integer,default=0)
92 95
93 96 class Task(db.Model):
94 97 '''
... ... @@ -111,9 +114,11 @@ class Task(db.Model):
111 114 #1：入库任务
112 115 #2：抽稀任务
113 116 #3：数据库刷新任务
114   - #4：影像金字塔任务
115   - #5：数据下载任务
116   - task_type=Column(Integer)
  117 + #4：数据下载任务
  118 + #5：影像金字塔任务
  119 + task_type = Column(Integer)
  120 + #任务的pid，用于kill
  121 + task_pid = Column(Integer)
117 122 creator = Column(Text)
118 123 file_name = Column(Text)
119 124 relate_processes = relationship('Process', backref='relate_task', lazy='dynamic')
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2020/9/4
  4 +#email: nheweijun@sina.com
  5 +
  6 +from ..models import db,Task,Table
  7 +
  8 +
  9 +from app.util.component.ApiTemplate import ApiTemplate
  10 +import os
  11 +import signal
  12 +import platform
  13 +
  14 +
  15 +class Api(ApiTemplate):
  16 + api_name = "停止任务"
  17 + def para_check(self):
  18 + pass
  19 +
  20 + def process(self):
  21 + res = {}
  22 + try:
  23 + task_guid = self.para.get("task_guid")
  24 + task = Task.query.filter_by(guid=task_guid).one_or_none()
  25 + pid = task.task_pid
  26 + if platform.system().lower().__contains__("windows"):
  27 + os.popen('taskkill.exe /pid:' + str(pid))
  28 + else:
  29 + os.kill(pid,signal.SIGILL)
  30 + #处理kill任务后的事情
  31 + self.fix_task()
  32 + res["msg"] = "Kill成功！"
  33 + res["result"] = True
  34 + except Exception as e:
  35 + db.session.rollback()
  36 + raise e
  37 + return res
  38 +
  39 +
  40 + def fix_task(self,task):
  41 + if task.task_type==1:
  42 + pass
  43 + if task.task_type==2:
  44 + table = Table.query.filter_by(guid=task.table_guid).one_or_none()
  45 + if len(table.relate_table_vacuates.all())>0:
  46 + Table.query.filter_by(guid=task.table_guid).update({"is_vacuate":1})
  47 + if task.task_type==3:
  48 + pass
  49 + if task.task_type==3:
  50 + pass
  51 +
  52 + db.session.commit()
  53 +
  54 + api_doc = {
  55 + "tags": ["任务接口"],
  56 + "parameters": [
  57 + {"name": "task_guid",
  58 + "in": "formData",
  59 + "type": "string"},
  60 + ],
  61 + "responses": {
  62 + 200: {
  63 + "schema": {
  64 + "properties": {
  65 + }
  66 + }
  67 + }
  68 + }
  69 + }
\ No newline at end of file
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/11/2
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +class DmpProject:
  8 + pass
\ No newline at end of file
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/11/2
  4 +#email: nheweijun@sina.com
  5 +
  6 +from .DmpProject import DmpProject
  7 +
  8 +
  9 +class DmpServerProject:
  10 +
  11 + project:DmpProject = None
  12 + flag : int = None
\ No newline at end of file
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/11/2
  4 +#email: nheweijun@sina.com
  5 +
  6 +from .DmpServerProject import DmpServerProject
  7 +
  8 +class ImageServer:
  9 + dmp_server_projects = []
  10 +
  11 +
  12 +
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/11/2
  4 +#email: nheweijun@sina.com
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/11/2
  4 +#email: nheweijun@sina.com
... ...
... ... @@ -9,6 +9,7 @@ from app.util import BlueprintApi
9 9 from . import service_register
10 10 from . import service_exist_type
11 11 from . import service_list
  12 +from . import service_list_base
12 13 from . import service_delete
13 14 from . import service_state
14 15 from . import service_info
... ... @@ -59,6 +60,16 @@ class DataManager(BlueprintApi):
59 60 """
60 61 return service_list.Api().result
61 62
  63 +
  64 + @staticmethod
  65 + @bp.route('/BaseMapList', methods=['POST'])
  66 + @swag_from(service_list_base.Api.api_doc)
  67 + def api_service_list_base():
  68 + """
  69 + 底图服务列表
  70 + """
  71 + return service_list_base.Api().result
  72 +
62 73 @staticmethod
63 74 @bp.route('/Info', methods=['POST'])
64 75 @swag_from(service_info.Api.api_doc)
... ...
... ... @@ -20,6 +20,7 @@ import traceback
20 20
21 21 from .util.ImageData import ImageData
22 22 from .util.Opencv import Opencv
  23 +from app.util.component.ModelVisitor import ModelVisitor
23 24
24 25 class Api(ApiTemplate):
25 26
... ... @@ -44,10 +45,8 @@ class Api(ApiTemplate):
44 45 if not zoo.connected:
45 46 zoo.start()
46 47
47   -
48 48 guid = self.para.get("guid")
49 49
50   -
51 50 width = int(self.para.get("width") if self.para.get("width") else 512)
52 51 height = int(self.para.get("height") if self.para.get("height") else 512)
53 52 format = self.para.get("format") if self.para.get("format") else "image/jpeg"
... ... @@ -75,8 +74,7 @@ class Api(ApiTemplate):
75 74 offset = ((query_extent[3] - query_extent[1]) - (query_extent[2] - query_extent[0])) / 2.0
76 75 query_extent = [query_extent[0] - offset, query_extent[1], query_extent[2] + offset, query_extent[3]]
77 76
78   -
79   - image_data = ImageData(image_server, image)
  77 + image_data = ImageData(image_server, ModelVisitor.object_to_json(image))
80 78
81 79 pixel_array_t = image_data.get_data(query_extent, bands, height, width)
82 80
... ...
... ... @@ -12,29 +12,23 @@ class Api(ApiTemplate):
12 12 api_name = "获取数据服务器列表"
13 13
14 14 def process(self):
15   - from app import GLOBAL_DIC
16 15
17 16 # 返回结果
18 17 res = {}
  18 + res["data"] = []
19 19 try:
20   - zoo = GLOBAL_DIC.get("zookeeper")
21   - if zoo is None:
22   - zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100)
23   - zoo.start()
24   - GLOBAL_DIC["zookeeper"] = zoo
25   - else :
26   - if not zoo.connected:
27   - zoo.start()
  20 + zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=1)
  21 + zoo.start()
28 22 sers = zoo.get_children("/rpc")
29   - res["data"] = []
  23 +
30 24 for p in sers:
31 25 bl = str( zoo.get("/rpc/{}".format(p))[0] , encoding = "utf-8")
32 26 res["data"].append({"name": p, "belong": bl})
33   -
34   - res["data"].append({"name":"本地服务器","belong":""})
35   - res["result"] = True
  27 + zoo.close()
36 28 except Exception as e:
37   - raise e
  29 + pass
  30 + res["data"].append({"name": "本地服务器", "belong": ""})
  31 + res["result"] = True
38 32 return res
39 33
40 34 api_doc = {
... ...
... ... @@ -4,15 +4,20 @@
4 4 #email: nheweijun@sina.com
5 5
6 6
7   -
8 7 from app.util.component.ApiTemplate import ApiTemplate
9 8 from ..models import Service,ImageService,Image,db,ServiceFunction
10 9 import datetime
11 10 import json
12   -from .image_service_register import Api as RegisterApi
13 11 import configure
14 12 import uuid
15   -
  13 +import os
  14 +import cv2
  15 +import numpy
  16 +import random
  17 +from kazoo.client import KazooClient
  18 +from .util.ImageData import ImageData
  19 +from .util.MyThread import MyThread
  20 +from app.util.component.ModelVisitor import ModelVisitor
16 21 class Api(ApiTemplate):
17 22
18 23 api_name = "修改影像服务"
... ... @@ -55,8 +60,8 @@ class Api(ApiTemplate):
55 60 db.session.add(service_function)
56 61
57 62 #修改影像
  63 + image_service_exetent = []
58 64 if image_guids:
59   - image_service_exetent = []
60 65 imgservice:ImageService = image_service.one_or_none()
61 66 imgservice.images = []
62 67
... ... @@ -72,30 +77,146 @@ class Api(ApiTemplate):
72 77 image_service_exetent[2] = max(image_extent[2], image_service_exetent[2])
73 78 image_service_exetent[3] = max(image_extent[3], image_service_exetent[3])
74 79 imgservice.images.append(image)
75   -
76 80 image_update["extent"] = json.dumps(image_service_exetent)
77 81
  82 + if image_update:
  83 + image_service.update(image_update)
  84 +
78 85 if service_update or image_update or functions:
79 86 service_update["update_time"] = this_time
  87 +
80 88 if image_guids:
81   - register_api = RegisterApi()
82   - overview_file = register_api.get_overview(service, image_service)
  89 + overview_file = self.get_overview(image_guids,image_service_exetent)
83 90 service_update["overview"] = "http://{}/API/Service/Overview/{}".format(configure.deploy_ip_host, overview_file)
84 91
85 92 service.update(service_update)
86 93
87   - if image_update:
88   - image_service.update(image_update)
89   -
90 94 db.session.commit()
91   -
92 95 res["result"] = True
  96 +
93 97 except Exception as e:
94 98 db.session.rollback()
95 99 raise e
96 100
97 101 return res
98 102
  103 + def get_overview(self,guids,image_service_exetent):
  104 + from app import GLOBAL_DIC
  105 +
  106 + intersect_image = Image.query.filter(Image.guid.in_(guids.split(","))).all()
  107 + query_extent = image_service_exetent
  108 +
  109 + if query_extent[2] - query_extent[0] > query_extent[3] - query_extent[1]:
  110 + offset = ((query_extent[2] - query_extent[0]) - (query_extent[3] - query_extent[1])) / 2.0
  111 + query_extent = [query_extent[0], query_extent[1] - offset, query_extent[2],
  112 + query_extent[3] + offset]
  113 + else:
  114 + offset = ((query_extent[3] - query_extent[1]) - (query_extent[2] - query_extent[0])) / 2.0
  115 + query_extent = [query_extent[0] - offset, query_extent[1], query_extent[2] + offset,
  116 + query_extent[3]]
  117 +
  118 + height = 512
  119 + width = 512
  120 +
  121 +
  122 + # 缓存zookeeper
  123 + zoo = GLOBAL_DIC.get("zookeeper")
  124 + if zoo is None:
  125 + zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100)
  126 + zoo.start()
  127 + GLOBAL_DIC["zookeeper"] = zoo
  128 + else:
  129 + if not zoo.connected:
  130 + zoo.start()
  131 +
  132 + servers = GLOBAL_DIC.get("servers")
  133 + if servers is None:
  134 + servers = zoo.get_children("/rpc")
  135 + servers.append("本地服务器")
  136 + GLOBAL_DIC["servers"] = servers
  137 + else:
  138 + servers = GLOBAL_DIC.get("servers")
  139 +
  140 +
  141 + if len(intersect_image) > 1:
  142 +
  143 + # 结果矩阵
  144 + empty_list = [numpy.zeros((height, width), dtype=int) + 65536,
  145 + numpy.zeros((height, width), dtype=int) + 65536,
  146 + numpy.zeros((height, width), dtype=int) + 65536]
  147 +
  148 + pixel_array = numpy.zeros((height, width, 3), dtype=int)
  149 + thread_list = []
  150 +
  151 + for image in intersect_image:
  152 + # 该影像的服务器，随机选取一个
  153 + image_servers = image.server.split(",")
  154 + image_servers = [ser for ser in image_servers if ser in servers]
  155 + if len(image_servers) > 0:
  156 + indx = int(random.random() * len(image_servers))
  157 + image_server = image_servers[indx]
  158 + else:
  159 + image_server = "None"
  160 + bands = json.loads(image.band_view)
  161 +
  162 + image_data = ImageData(image_server, ModelVisitor.object_to_json(image))
  163 +
  164 + thread: MyThread = MyThread(image_data.get_data, args=(query_extent, bands, height, width))
  165 + thread.start()
  166 + thread_list.append(thread)
  167 +
  168 + for thread in thread_list:
  169 + thread.join()
  170 + data = thread.get_result()
  171 +
  172 + # 掩膜在中央接口生成,合图
  173 + mask = numpy.zeros((height, width), dtype=int)
  174 + mask2 = numpy.zeros((height, width), dtype=int)
  175 + jizhun = data[:, :, 0]
  176 + mask[jizhun == 65536] = 1
  177 + mask[jizhun != 65536] = 0
  178 + mask2[jizhun == 65536] = 0
  179 + mask2[jizhun != 65536] = 1
  180 + # 掩膜计算
  181 + for i, d in enumerate(empty_list):
  182 + empty_list[i] = empty_list[i] * mask + data[:, :, i] * mask2
  183 +
  184 + for ii in [0, 1, 2]:
  185 + # opencv 颜色排序为GBR
  186 + pixel_array[:, :, 2 - ii] = empty_list[ii]
  187 +
  188 +
  189 + elif len(intersect_image) == 1:
  190 + # 该影像的服务器，随机选取一个
  191 + image = intersect_image[0]
  192 + image_servers = image.server.split(",")
  193 + image_servers = [ser for ser in image_servers if ser in servers]
  194 + if len(image_servers) > 0:
  195 + indx = int(random.random() * len(image_servers))
  196 + image_server = image_servers[indx]
  197 + else:
  198 + image_server = "None"
  199 +
  200 + bands = json.loads(image.band_view)
  201 +
  202 + image_data = ImageData(image_server,ModelVisitor.object_to_json(image))
  203 +
  204 + pixel_array_t = image_data.get_data(query_extent, bands, height, width)
  205 +
  206 + pixel_array = numpy.zeros((height, width, 3), dtype=int)
  207 + for ii in [0, 1, 2]:
  208 + # opencv 颜色排序为GBR
  209 + pixel_array[:, :, 2 - ii] = pixel_array_t[:, :, ii]
  210 + else:
  211 + # 结果矩阵
  212 + pixel_array = numpy.zeros((height, width, 3), dtype=int) + 65536
  213 +
  214 + dir_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "overview")
  215 + gid = uuid.uuid1().__str__()
  216 + store_file = os.path.join(dir_path, "{}.jpg".format(gid))
  217 + cv2.imwrite(store_file, pixel_array, [cv2.IMWRITE_JPEG_QUALITY, 30])
  218 +
  219 + return "{}.jpg".format(gid)
99 220
100 221 api_doc = {
101 222 "tags": ["影像接口"],
... ...
... ... @@ -32,23 +32,42 @@ class Api(ApiTemplate):
32 32 res = {}
33 33
34 34 try:
35   - guids = self.para.get("guids").split(",")
  35 + guids = self.para.get("image_guids").split(",")
36 36 name = self.para.get("name")
37 37 functions = self.para.get("functions")
38 38
39 39
40 40 image_service_guid = uuid.uuid1().__str__()
41 41 service_guid = uuid.uuid1().__str__()
42   - service_function_guid = uuid.uuid1().__str__()
43   -
44 42
45 43 this_time = datetime.datetime.now()
  44 +
  45 + service = Service(
  46 + guid = service_guid,
  47 + name = name,
  48 + title = self.para.get("title"),
  49 + state = 1,
  50 + create_time = this_time,
  51 + update_time = this_time,
  52 + description = self.para.get("description"),
  53 + #node = Column(Integer),
  54 + type = self.para.get("type"),
  55 + catalog_guid = self.para.get("catalog_guid")
  56 + )
  57 +
  58 + db.session.add(service)
  59 +
  60 +
46 61 image_service = ImageService(guid=image_service_guid,
47 62 name=name,
48 63 create_time=this_time,
49 64 scheme_guid=self.para.get("scheme_guid"),
50 65 crs = self.para.get("crs"),
51   - service_guid=service_guid)
  66 + service_guid=service_guid
  67 + )
  68 +
  69 + db.session.add(image_service)
  70 +
52 71 image_service_exetent = []
53 72
54 73 for g in guids:
... ... @@ -62,35 +81,18 @@ class Api(ApiTemplate):
62 81 image_service_exetent[1] = min(image_extent[1], image_service_exetent[1])
63 82 image_service_exetent[2] = max(image_extent[2], image_service_exetent[2])
64 83 image_service_exetent[3] = max(image_extent[3], image_service_exetent[3])
  84 +
65 85 image_service.images.append(image)
66 86 image_service.extent = json.dumps(image_service_exetent)
67 87
68 88
69   -
70   - service = Service(
71   - guid = service_guid,
72   - name = name,
73   - title = self.para.get("title"),
74   - state = 1,
75   - create_time = this_time,
76   - update_time = this_time,
77   - description = self.para.get("description"),
78   - #node = Column(Integer),
79   - type = self.para.get("type"),
80   - catalog_guid = self.para.get("catalog_guid")
81   - )
82   -
83   -
84   -
85 89 for type in functions.split(","):
  90 + service_function_guid = uuid.uuid1().__str__()
86 91 service_function = ServiceFunction(guid=service_function_guid,
87 92 service_guid=service_guid,
88 93 type=type)
89 94 db.session.add(service_function)
90 95
91   - db.session.add(service)
92   - db.session.add(image_service)
93   -
94 96
95 97 #获得overview
96 98 overview_file = self.get_overview(service,image_service)
... ... @@ -163,7 +165,7 @@ class Api(ApiTemplate):
163 165 "description": "[地图服务，切片服务，影像服务]"},
164 166
165 167 # 影像参数
166   - {"name": "guids",
  168 + {"name": "image_guids",
167 169 "in": "formData",
168 170 "type": "string",
169 171 "description": "[影像服务]影像guids，以英文逗号相隔"},
... ...
... ... @@ -19,8 +19,8 @@ from .util.MyThread import MyThread
19 19 from .util.ImageData import ImageData
20 20 from .util.Opencv import Opencv
21 21 from .util.Cache import Cache
22   -
23   -
  22 +from app.util.component.ModelVisitor import ModelVisitor
  23 +from app.util.component.ParameterUtil import StructurePrint
24 24 class Api(ApiTemplate):
25 25
26 26 api_name = "发布服务时预览"
... ... @@ -91,7 +91,7 @@ class Api(ApiTemplate):
91 91 image_server = "None"
92 92 bands = json.loads(image.band_view)
93 93
94   - image_data = ImageData(image_server, image)
  94 + image_data = ImageData(image_server, ModelVisitor.object_to_json(image))
95 95
96 96 thread: MyThread = MyThread(image_data.get_data, args=(extent,bands,height,width))
97 97 thread.start()
... ... @@ -132,7 +132,7 @@ class Api(ApiTemplate):
132 132
133 133 bands = json.loads(image.band_view)
134 134
135   - image_data = ImageData(image_server, image)
  135 + image_data = ImageData(image_server,ModelVisitor.object_to_json(image))
136 136
137 137 pixel_array_t = image_data.get_data(extent,bands,height,width)
138 138 pixel_array = numpy.zeros((height, width, 3), dtype=int)
... ... @@ -151,10 +151,8 @@ class Api(ApiTemplate):
151 151 return Response(im_data, mimetype=image_type.lower())
152 152
153 153 except Exception as e:
154   - print(traceback.format_exc())
155   - result["result"] = False
156   - result["message"] = e.__str__()
157   - return result
  154 + raise e
  155 +
158 156
159 157 def determin_intersect(self, extent1, extent2):
160 158 if extent2[2] < extent1[0] or extent2[0] > extent1[2] or extent2[1] > extent1[
... ...
... ... @@ -9,6 +9,8 @@ import configure
9 9 import time
10 10 from app.modules.service.models import ImageService
11 11 from app.modules.service.models import TileScheme,Service
  12 +from app.util.component.ModelVisitor import ModelVisitor
  13 +
12 14 import json
13 15
14 16 class Cache:
... ... @@ -21,7 +23,7 @@ class Cache:
21 23 # 缓存zookeeper
22 24 zoo = GLOBAL_DIC.get("zookeeper")
23 25 if zoo is None:
24   - zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100)
  26 + zoo: KazooClient = KazooClient(hosts=configure.zookeeper, timeout=1)
25 27 zoo.start()
26 28 GLOBAL_DIC["zookeeper"] = zoo
27 29 else:
... ... @@ -54,18 +56,23 @@ class Cache:
54 56 image_service: ImageService = ImageService.query.filter_by(guid=guid_or_name).one_or_none()
55 57 else:
56 58 image_service: ImageService = ImageService.query.filter_by(name=guid_or_name).one_or_none()
  59 +
57 60 images = image_service.images.all()
58 61
59 62 if image_service.scheme_guid:
60 63 scheme: TileScheme = TileScheme.query.filter_by(guid=image_service.scheme_guid).one_or_none()
61   - GLOBAL_DIC[guid_or_name] = {"service": image_service, "images": images,
62   - "scheme": json.loads(scheme.parameter)}
  64 +
  65 + GLOBAL_DIC[guid_or_name] = {"service": ModelVisitor.object_to_json(image_service),
  66 + "images": ModelVisitor.objects_to_jsonarray(images),
  67 + "scheme": json.loads(scheme.parameter)}
63 68 else:
64 69
65   - GLOBAL_DIC[guid_or_name] = {"service": image_service, "images": images}
  70 + GLOBAL_DIC[guid_or_name] = {"service": ModelVisitor.object_to_json(image_service),
  71 + "images": ModelVisitor.objects_to_jsonarray(images)}
  72 +
  73 +
66 74 GLOBAL_DIC["service_updatetime"] = time.time()
67 75 image_service_info = GLOBAL_DIC[guid_or_name]
68   -
69 76 else:
70 77 image_service_info = GLOBAL_DIC[guid_or_name]
71 78 else:
... ...
... ... @@ -41,9 +41,9 @@ class ImageData:
41 41
42 42 # 需要做thrift连接的缓存，连接池
43 43 thrift_connect = ThriftConnect(self.image_server)
44   - image_extent = self.image.extent
  44 + image_extent = self.image.get("extent")
45 45
46   - data = thrift_connect.client.getData(self.image.path, extent, json.loads(image_extent), bands, width, height)
  46 + data = thrift_connect.client.getData(self.image.get("path"), extent, json.loads(image_extent), bands, width, height)
47 47
48 48 thrift_connect.close()
49 49
... ... @@ -65,9 +65,9 @@ class ImageData:
65 65
66 66 # 需要做thrift连接的缓存，连接池
67 67 thrift_connect = ThriftConnect(image_server)
68   - image_extent = image.extent
  68 + image_extent = image.get("extent")
69 69
70   - data = thrift_connect.client.getData(image.path, extent, json.loads(image_extent), bands, width, height)
  70 + data = thrift_connect.client.getData(image.get("path"), extent, json.loads(image_extent), bands, width, height)
71 71
72 72 thrift_connect.close()
73 73
... ... @@ -84,14 +84,14 @@ class ImageData:
84 84 '''
85 85 pixel_array = numpy.zeros((height, width, 3), dtype=int)
86 86 ceng = 0
87   - img: Dataset = gdal.Open(self.image.path, 0)
  87 + img: Dataset = gdal.Open(self.image.get("path"), 0)
88 88
89 89 for band in bands:
90 90
91 91 # 自决定金字塔等级
92 92 xysize = [img.RasterXSize, img.RasterYSize]
93 93
94   - origin_extent = json.loads(self.image.extent)
  94 + origin_extent = json.loads(self.image.get("extent"))
95 95 band_data: Band = img.GetRasterBand(band)
96 96
97 97 max_level = band_data.GetOverviewCount()
... ...
... ... @@ -52,7 +52,7 @@ class ImageWMSServer(ImageServer):
52 52 extent = [float(x) for x in bbox.split(",")]
53 53
54 54 intersect_image = [im for im in image_service_info["images"] if
55   - self.determin_intersect(json.loads(im.extent), extent)]
  55 + self.determin_intersect(json.loads(im.get("extent")), extent)]
56 56
57 57 if len(intersect_image) > 1:
58 58
... ... @@ -66,14 +66,14 @@ class ImageWMSServer(ImageServer):
66 66
67 67 for image in intersect_image:
68 68 # 该影像的服务器，随机选取一个
69   - image_servers = image.server.split(",")
  69 + image_servers = image.get("server").split(",")
70 70 image_servers = [ser for ser in image_servers if ser in servers]
71 71 if len(image_servers) > 0:
72 72 indx = int(random.random() * len(image_servers))
73 73 image_server = image_servers[indx]
74 74 else:
75 75 image_server = "None"
76   - bands = json.loads(image.band_view)
  76 + bands = json.loads(image.get("band_view"))
77 77
78 78 image_data = ImageData(image_server, image)
79 79
... ... @@ -105,7 +105,7 @@ class ImageWMSServer(ImageServer):
105 105 elif len(intersect_image) == 1:
106 106 # 该影像的服务器，随机选取一个
107 107 image = intersect_image[0]
108   - image_servers = image.server.split(",")
  108 + image_servers = image.get("server").split(",")
109 109 image_servers = [ser for ser in image_servers if ser in servers]
110 110 if len(image_servers) > 0:
111 111 indx = int(random.random() * len(image_servers))
... ... @@ -113,7 +113,7 @@ class ImageWMSServer(ImageServer):
113 113 else:
114 114 image_server = "None"
115 115
116   - bands = json.loads(image.band_view)
  116 + bands = json.loads(image.get("band_view"))
117 117
118 118 image_data = ImageData(image_server, image)
119 119
... ... @@ -232,19 +232,19 @@ class ImageWMSServer(ImageServer):
232 232 </Capability>
233 233 </WMS_Capabilities>'''
234 234
235   - extent = json.loads(image_service.extent)
236   - xml = xml.format(service_title=image_service.name,
237   - service_name=image_service.name,
  235 + extent = json.loads(image_service.get("extent"))
  236 + xml = xml.format(service_title=image_service.get("name"),
  237 + service_name=image_service.get("name"),
238 238 abstract="None",
239 239 crs="ESPG:4326",
240   - layer_name=image_service.name,
241   - layer_title=image_service.name,
  240 + layer_name=image_service.get("name"),
  241 + layer_title=image_service.get("name"),
242 242 maxx=extent[2],
243 243 maxy=extent[3],
244 244 minx=extent[0],
245 245 miny=extent[1],
246 246 url="http://{}/API/Service/Image/WMS?guid={}".format(configure.deploy_ip_host,
247   - image_service.guid))
  247 + image_service.get("guid")))
248 248
249 249 r = Response(response=xml, status=200, mimetype="application/xml")
250 250 r.headers["Content-Type"] = "text/xml; charset=utf-8"
... ...
... ... @@ -18,6 +18,7 @@ import random
18 18 import copy
19 19 import configure
20 20 from app.util.component.SliceScheme import SliceScheme
  21 +from app.util.component.ModelVisitor import ModelVisitor
21 22 import threading
22 23 from .ImageServer import ImageServer
23 24
... ... @@ -44,7 +45,7 @@ class ImageWMTSServer(ImageServer):
44 45
45 46 re = parameter.get("request")
46 47 if re and re.__eq__("GetCapabilities"):
47   - service = Service.query.filter_by(guid=image_service_info["service"].service_guid).one_or_none()
  48 + service = Service.query.filter_by(guid=image_service_info["service"].get("service_guid")).one_or_none()
48 49 return self.get_wmts_capabilities(image_service_info["service"], service)
49 50
50 51 if parameter.get("tilematrix"):
... ... @@ -67,7 +68,7 @@ class ImageWMTSServer(ImageServer):
67 68 # 多线程获取分布式数据
68 69
69 70 intersect_image = [im for im in image_service_info["images"] if
70   - self.determin_intersect(json.loads(im.extent), extent)]
  71 + self.determin_intersect(json.loads(im.get("extent")), extent)]
71 72
72 73 if len(intersect_image) > 1:
73 74
... ... @@ -79,7 +80,7 @@ class ImageWMTSServer(ImageServer):
79 80 for image in intersect_image:
80 81
81 82 # 该影像的服务器，随机选取一个
82   - image_servers = image.server.split(",")
  83 + image_servers = image.get("server").split(",")
83 84 image_servers = [ser for ser in image_servers if ser in servers]
84 85 if len(image_servers) > 0:
85 86 indx = int(random.random() * len(image_servers))
... ... @@ -87,7 +88,7 @@ class ImageWMTSServer(ImageServer):
87 88 else:
88 89 image_server = "None"
89 90
90   - bands = json.loads(image.band_view)
  91 + bands = json.loads(image.get("band_view"))
91 92
92 93 image_data = ImageData(image_server, image)
93 94
... ... @@ -121,7 +122,7 @@ class ImageWMTSServer(ImageServer):
121 122 elif len(intersect_image) == 1:
122 123 # 该影像的服务器，随机选取一个
123 124 image = intersect_image[0]
124   - image_servers = image.server.split(",")
  125 + image_servers = image.get("server").split(",")
125 126 # 判断可用服务器
126 127 image_servers = [ser for ser in image_servers if ser in servers]
127 128 if len(image_servers) > 0:
... ... @@ -130,7 +131,7 @@ class ImageWMTSServer(ImageServer):
130 131 else:
131 132 image_server = "None"
132 133 # image_server = image_servers[0]
133   - bands = json.loads(image.band_view)
  134 + bands = json.loads(image.get("band_view"))
134 135
135 136 image_data = ImageData(image_server, image)
136 137 pixel_array_t: numpy.ndarray = image_data.get_data(extent, bands, height, width)
... ... @@ -149,11 +150,11 @@ class ImageWMTSServer(ImageServer):
149 150 return Response(im_data, mimetype=image_type.lower())
150 151
151 152
152   - def get_wmts_capabilities(self, image_service: ImageService, service: Service):
153   - tile_scheme: TileScheme = TileScheme.query.filter_by(guid=image_service.scheme_guid).one_or_none()
  153 + def get_wmts_capabilities(self, image_service, service: Service):
  154 + tile_scheme: TileScheme = TileScheme.query.filter_by(guid=image_service.get("scheme_guid")).one_or_none()
154 155 if not tile_scheme:
155 156 raise Exception("切片方案不存在！")
156   -
  157 + tile_scheme = ModelVisitor.object_to_json(tile_scheme)
157 158 xml = '''<Capabilities xmlns="http://www.opengis.net/wmts/1.0" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:gml="http://www.opengis.net/gml" xsi:schemaLocation="http://www.opengis.net/wmts/1.0 http://schemas.opengis.net/wmts/1.0/wmtsGetCapabilities_response.xsd" version="1.0.0">
158 159 <!-- Service Identification -->
159 160 <ows:ServiceIdentification>
... ... @@ -267,21 +268,21 @@ class ImageWMTSServer(ImageServer):
267 268 '''
268 269
269 270 tile_matrix = ""
270   - top_left = tile_scheme.top_left
271   - for level in json.loads(tile_scheme.levels):
  271 + top_left = tile_scheme.get("top_left")
  272 + for level in json.loads(tile_scheme.get("levels")):
272 273 tile_matrix = "{}{}".format(tile_matrix, tile_matrix_each.format(lev=level["level"],
273 274 scale=level["scale"],
274 275 top_left=top_left,
275   - cols=tile_scheme.cols,
276   - rows=tile_scheme.rows))
  276 + cols=tile_scheme.get("cols"),
  277 + rows=tile_scheme.get("rows")))
277 278
278   - extent = json.loads(image_service.extent)
  279 + extent = json.loads(image_service.get("extent"))
279 280
280 281 xml = xml.format(
281 282 capabilities_url="http://{}/API/Service/Image/Capabilities?guid={}".format(configure.deploy_ip_host,
282   - image_service.guid),
283   - tile_url="http://{}/API/Service/Image/Tile?guid={}".format(configure.deploy_ip_host, image_service.guid),
284   - crs=tile_scheme.crs,
  283 + image_service.get("guid")),
  284 + tile_url="http://{}/API/Service/Image/Tile?guid={}".format(configure.deploy_ip_host, image_service.get("guid")),
  285 + crs=tile_scheme.get("crs"),
285 286 xmin=extent[0],
286 287 ymin=extent[1],
287 288 xmax=extent[2],
... ... @@ -289,11 +290,11 @@ class ImageWMTSServer(ImageServer):
289 290 # TileMatrix = "{TileMatrix}",
290 291 # TileRow = "{TileRow}",
291 292 # TileCol = "{TileCol}",
292   - guid=image_service.guid,
  293 + guid=image_service.get("guid"),
293 294 title=service.title,
294   - tile_title=tile_scheme.name,
295   - tile_name=tile_scheme.name,
296   - tile_description=tile_scheme.description,
  295 + tile_title=tile_scheme.get("name"),
  296 + tile_name=tile_scheme.get("name"),
  297 + tile_description=tile_scheme.get("description"),
297 298 tile_matrix=tile_matrix
298 299 )
299 300
... ...
... ... @@ -6,7 +6,7 @@
6 6
7 7 from app.util.component.ApiTemplate import ApiTemplate
8 8 from app.util.component.ModelVisitor import ModelVisitor
9   -from .models import Service,ServiceFunction
  9 +from .models import Service,ServiceFunction,ImageService
10 10 from sqlalchemy import or_
11 11
12 12
... ... @@ -26,10 +26,14 @@ class Api(ApiTemplate):
26 26 name = self.para.get("name")
27 27 type = self.para.get("type")
28 28 function_type = self.para.get("function_type")
  29 + state = self.para.get("state")
29 30
30 31 catalog_guid = self.para.get("catalog_guid")
31 32
32 33 services = Service.query.order_by(Service.update_time.desc())
  34 +
  35 + if state:
  36 + services = services.filter_by(state=int(state))
33 37 if type:
34 38 services = services.filter_by(type=type)
35 39
... ... @@ -44,6 +48,11 @@ class Api(ApiTemplate):
44 48 services = services.filter(Service.title.like("%" + title + "%"))
45 49 if name:
46 50 services = services.filter(Service.name.like("%" + name + "%"))
  51 +
  52 + if onlyWMTS:
  53 + services = services.join(ServiceFunction).filter(ServiceFunction.type=="WMTS")
  54 +
  55 +
47 56 res["data"] = {}
48 57 res["data"]["count"] = services.count()
49 58 services = services.limit(page_size).offset(page_index * page_size).all()
... ... @@ -61,6 +70,9 @@ class Api(ApiTemplate):
61 70
62 71 return res
63 72
  73 + def judge(self,name):
  74 + return name.__contains__("演示")
  75 +
64 76 api_doc = {
65 77 "tags": ["服务接口"],
66 78 "parameters": [
... ... @@ -80,6 +92,11 @@ class Api(ApiTemplate):
80 92 "in": "formData",
81 93 "type": "string",
82 94 "description": "服务名"},
  95 + {"name": "state",
  96 + "in": "formData",
  97 + "type": "string",
  98 + "description": "状态",
  99 + "enum":[0,1]},
83 100 {"name": "type",
84 101 "in": "formData",
85 102 "type": "string",
... ...
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
  5 +
  6 +
  7 +from app.util.component.ApiTemplate import ApiTemplate
  8 +from app.util.component.ModelVisitor import ModelVisitor
  9 +from .models import Service,ServiceFunction,ImageService
  10 +from sqlalchemy import or_
  11 +from app.util.component.GeometryAdapter import GeometryAdapter
  12 +
  13 +import json
  14 +
  15 +class Api(ApiTemplate):
  16 +
  17 + api_name = "底图服务列表"
  18 +
  19 + def process(self):
  20 +
  21 + # 返回结果
  22 + res = {}
  23 +
  24 + try:
  25 + page_index = int(self.para.get("page_index", "0"))
  26 + page_size = int(self.para.get("page_size", "10"))
  27 + bbox = self.para.get("bbox")
  28 + state = self.para.get("state")
  29 +
  30 + services = Service.query.order_by(Service.update_time.desc())
  31 +
  32 + if state:
  33 + services = services.filter_by(state=int(state))
  34 +
  35 + services = services.join(ServiceFunction).filter(ServiceFunction.type=="WMTS")
  36 + services = services.all()
  37 +
  38 +
  39 + fit_services = []
  40 + if bbox:
  41 + g1 = GeometryAdapter.bbox_2_polygon([float(x) for x in bbox.split(",")])
  42 + for ser in services:
  43 + if ser.type.__eq__("切片服务"):
  44 + layer_exetent = ser.relate_tile_service.one_or_none().layer_extent
  45 +
  46 + g2 = GeometryAdapter.envelop_2_polygon([float(x) for x in layer_exetent.split(",")])
  47 + if g1.Intersect(g2):
  48 + fit_services.append(ser)
  49 + if ser.type.__eq__("影像服务"):
  50 + image_extent = ser.relate_image_service.one_or_none().extent
  51 + g2 = GeometryAdapter.bbox_2_polygon(json.loads(image_extent))
  52 + if g1.Intersect(g2):
  53 + fit_services.append(ser)
  54 + else:
  55 + fit_services = services
  56 +
  57 + res["data"] = {}
  58 + res["data"]["count"] = len(fit_services)
  59 +
  60 + res["data"]["list"] = ModelVisitor.objects_to_jsonarray(fit_services[page_index * page_size:(page_index+1) * page_size])
  61 +
  62 + for service_json in res["data"]["list"]:
  63 + service_json["functions"] = sorted(ModelVisitor.objects_to_jsonarray(ServiceFunction.query.filter_by(service_guid=service_json["guid"]).all()),
  64 + key=lambda x:x["type"])
  65 +
  66 + res["result"] = True
  67 +
  68 +
  69 + except Exception as e:
  70 + raise e
  71 +
  72 + return res
  73 +
  74 + def judge(self,name):
  75 + return name.__contains__("演示")
  76 +
  77 + api_doc = {
  78 + "tags": ["服务接口"],
  79 + "parameters": [
  80 + {"name": "page_index",
  81 + "in": "formData",
  82 + "type": "int",
  83 + "description": "页"},
  84 + {"name": "page_size",
  85 + "in": "formData",
  86 + "type": "int",
  87 + "description": "页大小"},
  88 +
  89 + {"name": "state",
  90 + "in": "formData",
  91 + "type": "string",
  92 + "description": "状态",
  93 + "enum":[0,1]},
  94 +
  95 + {"name": "bbox",
  96 + "in": "formData",
  97 + "type": "String",
  98 + "description": "x1,y1,x2,y2"},
  99 +
  100 + ],
  101 + "responses": {
  102 + 200: {
  103 + "schema": {
  104 + "properties": {
  105 + }
  106 + }
  107 + }
  108 + }
  109 + }
... ...
... ... @@ -60,7 +60,7 @@ class Api(ApiTemplate):
60 60 "description": "[地图服务，切片服务，影像服务]"},
61 61
62 62 #影像参数
63   - {"name": "guids",
  63 + {"name": "image_guids",
64 64 "in": "formData",
65 65 "type": "string",
66 66 "description": "[影像服务]影像guids，以英文逗号相隔"},
... ...
... ... @@ -144,7 +144,10 @@ class EntryDataVacuate:
144 144
145 145
146 146 for i in range(ds.GetLayerCount()):
  147 +
147 148 layer: Layer = ds.GetLayer(i)
  149 + if layer.GetName().lower() not in meta.get("layer").keys():
  150 + continue
148 151 is_success, new_layer_name = self.entry_one_layer(layer,this_task,meta)
149 152 new_layer_names.append(new_layer_name)
150 153 is_successes.append(is_success)
... ... @@ -330,7 +333,12 @@ class ThisTask:
330 333 is_vacuate=is_vacuate
331 334 )
332 335 # 删除遗留业务数据
333   - history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all()
  336 + try:
  337 + history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all()
  338 + except:
  339 + self.sys_session: Session = PGUtil.get_db_session(configure.SQLALCHEMY_DATABASE_URI)
  340 + history_table = self.sys_session.query(Table).filter_by(name=new_layer_name,database_guid=self.database.guid).all()
  341 +
334 342 if history_table:
335 343 for ht in history_table:
336 344 self.sys_session.delete(ht)
... ...
... ... @@ -56,8 +56,6 @@ class SliceScheme:
56 56 maxy = -detaxy * row + int(parameter.get("y"))
57 57 maxx = detaxy + minx
58 58 miny = -detaxy + maxy
59   -
60   -
61 59 return [minx,miny,maxx,maxy]
62 60
63 61
... ...
... ... @@ -5,6 +5,7 @@
5 5
6 6
7 7 class VacuateConf:
  8 + original_density = 4
8 9 least_vacuate_count = 20000
9 10 lonlat_gridsize = [0.00008, 0.00016, 0.00032, 0.00064, 0.00128, 0.00256, 0.00512, 0.01024, 0.02048,0.04096, 0.08192]
10 11 project_gridsize = [8,16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
\ No newline at end of file
... ...
1 1 # coding=utf-8
2   -
  2 +import logging
3 3 # 程序部署ip:host
4 4 deploy_ip_host = "172.26.40.105:8840"
5 5 # 系统数据库
... ... @@ -27,3 +27,5 @@ entry_data_thread = 3
27 27 scan_module = ["app.modules"] # API所在的模块
28 28 SECRET_KEY = b'_5#y2L"F4Q8z\n\xec]/'
29 29
  30 +log_level = logging.INFO
  31 +
... ...
... ... @@ -24,7 +24,7 @@ fi
24 24 #启动容器和apache
25 25 echo "正在启动容器..."
26 26 set="--privileged=true -e TZ="Asia/Shanghai" --restart=always -e ALLOW_IP_RANGE=0.0.0.0/0"
27   -docker run -d --name $dn $set -p $port:80 -v $curPath:/usr/src/app -v $curPath/httpd.conf:/etc/httpd/conf/httpd.conf dci/dmapmanager:4.0 /usr/sbin/init
  27 +docker run -d --name $dn $set -p $port:80 -v $curPath/logs/apache.error:/var/log/httpd/error_log -v $curPath:/usr/src/app -v $curPath/httpd.conf:/etc/httpd/conf/httpd.conf dci/dmapmanager:4.0 /usr/sbin/init
28 28 docker exec -d $dn systemctl start httpd
29 29 sleep 5
30 30 curl localhost:$port/release
... ...
... ... @@ -53,12 +53,6 @@ def query_thread():
53 53 ses.close()
54 54 # query_thread()
55 55 if __name__ == '__main__':
56   - time1 = time.time()
57   - ps=[]
58   - for i in range(20):
59   - t = multiprocessing.Process(target=query_thread)
60   - t.start()
61   - ps.append(t)
62   - for p in ps:
63   - p.join()
64   - print(time.time()-time1)
\ No newline at end of file
  56 + kk = [1,2,3,4,5,6,7,8,9,10,11]
  57 + print(len(kk))
  58 + print(kk[10:20])
\ No newline at end of file
... ...
请 注册 或 登录 后发表评论