提交 6ca42a95087454c4fc771c3a2d60256c4d0c0d33

作者 nheweijun
1 个父辈 25ceb357

项目结构调整,调整data和service 2个大模块

正在显示 87 个修改的文件 包含 179 行增加547 行删除
@@ -3,7 +3,6 @@ import decimal @@ -3,7 +3,6 @@ import decimal
3 from flask import Flask as _Flask 3 from flask import Flask as _Flask
4 from flask.json import JSONEncoder as _JSONEncoder 4 from flask.json import JSONEncoder as _JSONEncoder
5 from flask_cors import CORS 5 from flask_cors import CORS
6 -import time  
7 6
8 from sqlalchemy.sql.expression import true 7 from sqlalchemy.sql.expression import true
9 import configure 8 import configure
@@ -12,24 +11,14 @@ from app.util import find_class @@ -12,24 +11,14 @@ from app.util import find_class
12 from app.models import db, Table, InsertingLayerName, Database, DES, Task 11 from app.models import db, Table, InsertingLayerName, Database, DES, Task
13 from app.modules.auth.oauth2 import config_oauth, myCodeIDToken 12 from app.modules.auth.oauth2 import config_oauth, myCodeIDToken
14 from flasgger import Swagger 13 from flasgger import Swagger
15 -# from rtree import index  
16 import logging 14 import logging
17 -from sqlalchemy.orm import Session  
18 -import multiprocessing  
19 from app.util.component.EntryData import EntryData 15 from app.util.component.EntryData import EntryData
20 from app.util.component.EntryDataVacuate import EntryDataVacuate 16 from app.util.component.EntryDataVacuate import EntryDataVacuate
21 -import json  
22 import threading 17 import threading
23 -import traceback  
24 -from sqlalchemy import distinct  
25 -import uuid  
26 -from osgeo.ogr import DataSource  
27 -import datetime  
28 -from sqlalchemy import or_  
29 from app.util.component.StructuredPrint import StructurePrint 18 from app.util.component.StructuredPrint import StructurePrint
30 from app.util.component.PGUtil import PGUtil 19 from app.util.component.PGUtil import PGUtil
31 import os 20 import os
32 - 21 +from app.modules.data.io.data_entry_center import data_entry_center
33 22
34 class JSONEncoder(_JSONEncoder): 23 class JSONEncoder(_JSONEncoder):
35 """ 24 """
@@ -105,7 +94,6 @@ def create_app(): @@ -105,7 +94,6 @@ def create_app():
105 app.register_blueprint(api.bp) 94 app.register_blueprint(api.bp)
106 95
107 # 入库监测线程 96 # 入库监测线程
108 -  
109 @app.before_first_request 97 @app.before_first_request
110 def data_entry_process(): 98 def data_entry_process():
111 StructurePrint.print("start listen") 99 StructurePrint.print("start listen")
@@ -120,99 +108,3 @@ def create_app(): @@ -120,99 +108,3 @@ def create_app():
120 108
121 return app 109 return app
122 110
123 -  
124 -def data_entry_center():  
125 - running_dict = {}  
126 - sys_session: Session = PGUtil.get_db_session(  
127 - configure.SQLALCHEMY_DATABASE_URI)  
128 -  
129 - while True:  
130 -  
131 - try:  
132 - time.sleep(3)  
133 -  
134 - # 已经结束的进程 从监测中删除  
135 - remove_process = []  
136 -  
137 - # structured_print(running_dict.__len__().__str__())  
138 -  
139 - for process, layer_names in running_dict.items():  
140 - if not process.is_alive():  
141 - for l in layer_names:  
142 - inserted = sys_session.query(  
143 - InsertingLayerName).filter_by(name=l).one_or_none()  
144 - if inserted:  
145 - sys_session.delete(inserted)  
146 - sys_session.commit()  
147 - remove_process.append(process)  
148 - for process in remove_process:  
149 - running_dict.pop(process)  
150 -  
151 - # StructurePrint.print("listening...")  
152 -  
153 - # 入库进程少于阈值,开启入库进程  
154 -  
155 - inter_size = sys_session.query(  
156 - distinct(InsertingLayerName.task_guid)).count()  
157 -  
158 - if inter_size < configure.entry_data_thread:  
159 - # 锁表啊  
160 - ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by(  
161 - Task.create_time).with_lockmode("update").limit(1).one_or_none()  
162 - if ready_task:  
163 -  
164 - try:  
165 - parameter = json.loads(ready_task.parameter)  
166 - StructurePrint.print("检测到入库任务")  
167 - ready_task.state = 2  
168 - ready_task.process = "入库中"  
169 - sys_session.commit()  
170 -  
171 - metas: list = json.loads(  
172 - parameter.get("meta").__str__())  
173 - parameter["meta"] = metas  
174 -  
175 - database = sys_session.query(Database).filter_by(  
176 - guid=ready_task.database_guid).one_or_none()  
177 - pg_ds: DataSource = PGUtil.open_pg_data_source(  
178 - 1, DES.decode(database.sqlalchemy_uri))  
179 -  
180 - this_task_layer = []  
181 - for meta in metas:  
182 - overwrite = parameter.get("overwrite", "no")  
183 -  
184 - for layer_name_origin, layer_name in meta.get("layer").items():  
185 - origin_name = layer_name  
186 - no = 1  
187 -  
188 - while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none():  
189 - layer_name = origin_name + "_{}".format(no)  
190 - no += 1  
191 -  
192 - # 添加到正在入库的列表中  
193 - iln = InsertingLayerName(guid=uuid.uuid1().__str__(),  
194 - task_guid=ready_task.guid,  
195 - name=layer_name)  
196 -  
197 - sys_session.add(iln)  
198 - sys_session.commit()  
199 - this_task_layer.append(layer_name)  
200 - # 修改表名  
201 - meta["layer"][layer_name_origin] = layer_name  
202 -  
203 - pg_ds.Destroy()  
204 - entry_data_process = multiprocessing.Process(  
205 - target=EntryDataVacuate().entry, args=(parameter,))  
206 - entry_data_process.start()  
207 - running_dict[entry_data_process] = this_task_layer  
208 - except Exception as e:  
209 - sys_session.query(Task).filter_by(guid=ready_task.guid).update(  
210 - {"state": -1, "process": "入库失败"})  
211 - sys_session.commit()  
212 - StructurePrint.print(e.__str__(), "error")  
213 - else:  
214 - # 解表啊  
215 - sys_session.commit()  
216 - except Exception as e:  
217 - sys_session.commit()  
218 - StructurePrint.print(e.__str__(), "error")  
@@ -34,13 +34,14 @@ class DES(): @@ -34,13 +34,14 @@ class DES():
34 db = SQLAlchemy() 34 db = SQLAlchemy()
35 35
36 # 动态加载Model 36 # 动态加载Model
37 -module_dir = "modules"  
38 -model_pkg = "models"  
39 current_dir = os.path.abspath(os.path.dirname(__file__)) 37 current_dir = os.path.abspath(os.path.dirname(__file__))
40 -for pkg in glob.glob('%s/%s/*/%s' % (current_dir,module_dir,model_pkg)) :  
41 - module = os.path.basename(os.path.dirname(pkg))  
42 - base_name = os.path.basename(pkg).split('.')[0]  
43 - pkg_name = ".".join(["app.modules",module,base_name]) 38 +pkgs = list(glob.glob('%s/modules/*/models' % (current_dir)))
  39 +pkgs.extend(list(glob.glob('%s/modules/*/*/models' % (current_dir))))
  40 +
  41 +for pkg in pkgs :
  42 + pkg = os.path.normpath(pkg)
  43 + node_list = pkg.split(os.path.sep)
  44 + pkg_name = ".".join(node_list[node_list.index("app"):])
44 try: 45 try:
45 __import__(pkg_name) 46 __import__(pkg_name)
46 except: 47 except:
@@ -199,21 +200,21 @@ class InsertingLayerName(db.Model): @@ -199,21 +200,21 @@ class InsertingLayerName(db.Model):
199 name = Column(String(256)) 200 name = Column(String(256))
200 201
201 202
202 -# class Service(db.Model):  
203 -# '''  
204 -# '''  
205 -# __tablename__ = 'dmdms_service'  
206 -# guid = Column(String(256), primary_key=True)  
207 -# name=Column(String(256))  
208 -# alias = Column(String(256))  
209 -# #服务状态  
210 -# state= Column(Integer)  
211 -# create_time = Column(DateTime)  
212 -# update_time = Column(DateTime)  
213 -# #服务描述  
214 -# description = Column(Text)  
215 -# #服务节点  
216 -# node = Column(Integer)  
217 -# #服务缩略图  
218 -# overview = Column(Binary) 203 +class Service(db.Model):
  204 + '''
  205 + '''
  206 + __tablename__ = 'dmdms_service'
  207 + guid = Column(String(256), primary_key=True)
  208 + name=Column(String(256))
  209 + alias = Column(String(256))
  210 + #服务状态
  211 + state= Column(Integer)
  212 + create_time = Column(DateTime)
  213 + update_time = Column(DateTime)
  214 + #服务描述
  215 + description = Column(Text)
  216 + #服务节点
  217 + node = Column(Integer)
  218 + #服务缩略图
  219 + overview = Column(Binary)
219 220
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
@@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate @@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate
8 import uuid 8 import uuid
9 from . import database_test 9 from . import database_test
10 # from app.util import open_pg_data_source 10 # from app.util import open_pg_data_source
11 -from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn,Feature,Geometry 11 +from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn
12 from sqlalchemy.orm import Session 12 from sqlalchemy.orm import Session
13 from sqlalchemy import create_engine 13 from sqlalchemy import create_engine
14 from sqlalchemy.orm import sessionmaker 14 from sqlalchemy.orm import sessionmaker
@@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil @@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil
17 from app.util.component.SQLUtil import SQLUtil 17 from app.util.component.SQLUtil import SQLUtil
18 from app.util.component.GeometryAdapter import GeometryAdapter 18 from app.util.component.GeometryAdapter import GeometryAdapter
19 from app.util.component.StructuredPrint import StructurePrint 19 from app.util.component.StructuredPrint import StructurePrint
20 -import configure 20 +
  21 +
21 class Api(ApiTemplate): 22 class Api(ApiTemplate):
22 api_name = "注册数据库" 23 api_name = "注册数据库"
23 def process(self): 24 def process(self):
@@ -6,15 +6,13 @@ from flasgger import swag_from @@ -6,15 +6,13 @@ from flasgger import swag_from
6 from flask import Blueprint 6 from flask import Blueprint
7 from app.util import BlueprintApi 7 from app.util import BlueprintApi
8 8
9 -from flask import send_from_directory,current_app,send_file 9 +from flask import send_from_directory
10 import os 10 import os
11 -import shutil  
12 from . import data_download 11 from . import data_download
13 from . import get_meta 12 from . import get_meta
14 from . import data_entry_by_meta 13 from . import data_entry_by_meta
15 from . import get_data_list 14 from . import get_data_list
16 -from flask import after_this_request  
17 -import time 15 +
18 16
19 class DataManager(BlueprintApi): 17 class DataManager(BlueprintApi):
20 18
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
  5 +
  6 +import configure
  7 +from app.models import InsertingLayerName, Database, DES, Task
  8 +from sqlalchemy.orm import Session
  9 +import multiprocessing
  10 +from app.util.component.EntryDataVacuate import EntryDataVacuate
  11 +import json
  12 +from sqlalchemy import distinct
  13 +import uuid
  14 +from osgeo.ogr import DataSource
  15 +from app.util.component.StructuredPrint import StructurePrint
  16 +from app.util.component.PGUtil import PGUtil
  17 +import time
  18 +
  19 +def data_entry_center():
  20 + running_dict = {}
  21 + sys_session: Session = PGUtil.get_db_session(
  22 + configure.SQLALCHEMY_DATABASE_URI)
  23 +
  24 + while True:
  25 +
  26 + try:
  27 + time.sleep(3)
  28 +
  29 + # 已经结束的进程 从监测中删除
  30 + remove_process = []
  31 +
  32 + # structured_print(running_dict.__len__().__str__())
  33 +
  34 + for process, layer_names in running_dict.items():
  35 + if not process.is_alive():
  36 + for l in layer_names:
  37 + inserted = sys_session.query(
  38 + InsertingLayerName).filter_by(name=l).one_or_none()
  39 + if inserted:
  40 + sys_session.delete(inserted)
  41 + sys_session.commit()
  42 + remove_process.append(process)
  43 + for process in remove_process:
  44 + running_dict.pop(process)
  45 +
  46 + # StructurePrint.print("listening...")
  47 +
  48 + # 入库进程少于阈值,开启入库进程
  49 +
  50 + inter_size = sys_session.query(
  51 + distinct(InsertingLayerName.task_guid)).count()
  52 +
  53 + if inter_size < configure.entry_data_thread:
  54 + # 锁表啊
  55 + ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by(
  56 + Task.create_time).with_lockmode("update").limit(1).one_or_none()
  57 + if ready_task:
  58 +
  59 + try:
  60 + parameter = json.loads(ready_task.parameter)
  61 + StructurePrint.print("检测到入库任务")
  62 + ready_task.state = 2
  63 + ready_task.process = "入库中"
  64 + sys_session.commit()
  65 +
  66 + metas: list = json.loads(
  67 + parameter.get("meta").__str__())
  68 + parameter["meta"] = metas
  69 +
  70 + database = sys_session.query(Database).filter_by(
  71 + guid=ready_task.database_guid).one_or_none()
  72 + pg_ds: DataSource = PGUtil.open_pg_data_source(
  73 + 1, DES.decode(database.sqlalchemy_uri))
  74 +
  75 + this_task_layer = []
  76 + for meta in metas:
  77 + overwrite = parameter.get("overwrite", "no")
  78 +
  79 + for layer_name_origin, layer_name in meta.get("layer").items():
  80 + origin_name = layer_name
  81 + no = 1
  82 +
  83 + while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none():
  84 + layer_name = origin_name + "_{}".format(no)
  85 + no += 1
  86 +
  87 + # 添加到正在入库的列表中
  88 + iln = InsertingLayerName(guid=uuid.uuid1().__str__(),
  89 + task_guid=ready_task.guid,
  90 + name=layer_name)
  91 +
  92 + sys_session.add(iln)
  93 + sys_session.commit()
  94 + this_task_layer.append(layer_name)
  95 + # 修改表名
  96 + meta["layer"][layer_name_origin] = layer_name
  97 +
  98 + pg_ds.Destroy()
  99 + entry_data_process = multiprocessing.Process(
  100 + target=EntryDataVacuate().entry, args=(parameter,))
  101 + entry_data_process.start()
  102 + running_dict[entry_data_process] = this_task_layer
  103 + except Exception as e:
  104 + sys_session.query(Task).filter_by(guid=ready_task.guid).update(
  105 + {"state": -1, "process": "入库失败"})
  106 + sys_session.commit()
  107 + StructurePrint.print(e.__str__(), "error")
  108 + else:
  109 + # 解表啊
  110 + sys_session.commit()
  111 + except Exception as e:
  112 + sys_session.commit()
  113 + StructurePrint.print(e.__str__(), "error")
1 -# coding=utf-8  
2 -#author: 4N  
3 -#createtime: 2021/3/24  
4 -#email: nheweijun@sina.com  
5 -  
6 -from app.util import *  
7 -import traceback  
8 -from osgeo import gdal  
9 -from osgeo.gdal import *  
10 -from numpy import ndarray  
11 -import numpy  
12 -from flask import Response  
13 -import io  
14 -import os  
15 -from PIL import Image  
16 -  
17 -import time  
18 -import cv2  
19 -from app.modules.image.models import ImageService,Image  
20 -from app.models import db  
21 -from app.util.component.ApiTemplate import ApiTemplate  
22 -import uuid  
23 -from app.util.component.SliceScheme import SliceScheme  
24 -from app.util.component.FileProcess import FileProcess  
25 -from app.util.component.ParameterUtil import ParameterUtil  
26 -from app.util.component.GeometryAdapter import GeometryAdapter  
27 -import os  
28 -import json  
29 -from kazoo.client import KazooClient  
30 -from app import GLOBAL_DIC  
31 -from threading import Thread  
32 -from thrift.transport import TSocket  
33 -from thrift.transport import TTransport  
34 -from thrift.protocol import TBinaryProtocol  
35 -from .ImageDataService import ImageDataService  
36 -from flask import current_app  
37 -import gzip  
38 -import random  
39 -class Api(ApiTemplate):  
40 -  
41 - api_name = "切片"  
42 -  
43 - def __init__(self,guid,level, row, col):  
44 - super().__init__()  
45 - self.guid = guid  
46 - self.level = level  
47 - self.row = row  
48 - self.col = col  
49 -  
50 - def process(self):  
51 -  
52 - result = {}  
53 - parameter: dict = self.para  
54 -  
55 - try:  
56 - if parameter.get("guid"):  
57 - self.guid = parameter.get("guid")  
58 -  
59 - image_service = ImageService.query.filter_by(guid = self.guid).one_or_none()  
60 - images = image_service.images.all()  
61 -  
62 - zoo = GLOBAL_DIC.get("zookeeper")  
63 - if zoo is None:  
64 - zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100)  
65 - zoo.start()  
66 - GLOBAL_DIC["zookeeper"] = zoo  
67 - else :  
68 - if not zoo.connected:  
69 - zoo.start()  
70 -  
71 - bands = [1, 2, 3]  
72 -  
73 -  
74 - # 转换参数  
75 - parameter = ParameterUtil.to_lower(parameter)  
76 -  
77 -  
78 - if parameter.get("tilematrix"):  
79 - if parameter.get("tilematrix").__contains__(":"):  
80 - self.level = int(parameter.get("tilematrix").split(":")[-1])  
81 - else:  
82 - self.level = int(parameter.get("tilematrix"))  
83 - if parameter.get("tilerow"):  
84 - self.row = int(parameter.get("tilerow"))  
85 - if parameter.get("tilecol"):  
86 - self.col = int(parameter.get("tilecol"))  
87 -  
88 - image_type = parameter.get("format") if parameter.get("format") else "image/png"  
89 - quality = int(parameter.get("quality")) if parameter.get("quality") else 30  
90 - slice_para = json.loads(image_service.slice_scheme)  
91 - extent = SliceScheme.get_polygon(slice_para, self.level, self.row, self.col)  
92 -  
93 -  
94 -  
95 - # 多线程获取分布式数据  
96 -  
97 - intersect_image = [im for im in images if self.determin_intersect(json.loads(im.extent),extent)]  
98 -  
99 - pixel_array = numpy.zeros((256, 256,3), dtype=int)  
100 -  
101 - if len(intersect_image) > 1:  
102 -  
103 - # 结果矩阵  
104 - empty_list = [numpy.zeros((256,256), dtype=int) + 65536,  
105 - numpy.zeros((256,256), dtype=int) + 65536,  
106 - numpy.zeros((256,256), dtype=int) + 65536]  
107 -  
108 - pixel_array = numpy.zeros((256,256,3), dtype=int)  
109 - thread_list = []  
110 -  
111 - for image in intersect_image:  
112 - #该影像的服务器,随机选取一个  
113 - image_servers = image.server.split(",")  
114 - indx = int(random.random()*len(image_servers))  
115 - image_server = image_servers[indx]  
116 -  
117 - thread: MyThread = MyThread(self.get_data, args=(zoo,image_server,image,extent,bands))  
118 - thread.start()  
119 - thread_list.append(thread)  
120 -  
121 -  
122 - for thread in thread_list:  
123 - thread.join()  
124 - data = thread.get_result()  
125 -  
126 - # 掩膜在中央接口生成,合图  
127 - mask = numpy.zeros((256,256), dtype=int)  
128 - mask2 = numpy.zeros((256,256), dtype=int)  
129 - jizhun = data[:, :, 0]  
130 - mask[jizhun == 65536] = 1  
131 - mask[jizhun != 65536] = 0  
132 - mask2[jizhun == 65536] = 0  
133 - mask2[jizhun != 65536] = 1  
134 - # 掩膜计算  
135 - for i, d in enumerate(empty_list):  
136 - empty_list[i] = empty_list[i] * mask + data[:, :, i] * mask2  
137 - for ii in [0, 1, 2]:  
138 - # opencv 颜色排序为GBR  
139 - pixel_array[:, :, 2 - ii] = empty_list[ii]  
140 -  
141 - elif len(intersect_image) == 1:  
142 -  
143 - # 该影像的服务器,随机选取一个  
144 - image = intersect_image[0]  
145 - image_servers = image.server.split(",")  
146 - indx = int(random.random() * len(image_servers))  
147 - image_server = image_servers[indx]  
148 - pixel_array_t = self.get_data(zoo, image_server, image, extent, bands)  
149 - pixel_array = numpy.zeros((256,256, 3), dtype=int)  
150 - for ii in [0, 1, 2]:  
151 - # opencv 颜色排序为GBR  
152 - pixel_array[:, :, 2 - ii] = pixel_array_t[:, :, ii]  
153 -  
154 - else:  
155 - # 结果矩阵  
156 - pixel_array = numpy.zeros((256, 256, 3), dtype=int) + 65536  
157 -  
158 - #将图片生成在内存中,然后直接返回response  
159 - im_data = self.create_by_opencv(image_type, pixel_array, quality)  
160 -  
161 - return Response(im_data, mimetype=image_type.lower())  
162 -  
163 - except Exception as e:  
164 - print(traceback.format_exc())  
165 - result["state"] = -1  
166 - result["message"] = e.__str__()  
167 - return result  
168 -  
169 -  
170 - def determine_level(self,xysize,origin_extent,extent,max_level):  
171 - '''  
172 - 根据范围判断调用金字塔的哪一层  
173 - :param xysize:  
174 - :param origin_extent:  
175 - :param extent:  
176 - :param max_level:  
177 - :return:  
178 - '''  
179 - x = xysize[0]  
180 - y = xysize[1]  
181 - level = -1  
182 - pixel = x*y * (((extent[2]-extent[0])*(extent[3]-extent[1]))/((origin_extent[2]-origin_extent[0])*(origin_extent[3]-origin_extent[1])))  
183 - while pixel>100000 and level<max_level-1:  
184 - level+=1  
185 - x=x/2  
186 - y=y/2  
187 - pixel = x * y * (((extent[2] - extent[0]) * (extent[3] - extent[1])) / (  
188 - (origin_extent[2] - origin_extent[0]) * (origin_extent[3] - origin_extent[1])))  
189 - return level  
190 -  
191 - def create_by_opencv(self,image_type, pixel_array, quality):  
192 -  
193 - if image_type.__eq__("image/jpeg") or image_type.__eq__("image/jpg"):  
194 - r, buf = cv2.imencode(".jpg", pixel_array, [cv2.IMWRITE_JPEG_QUALITY, quality])  
195 - image_out = buf.tobytes()  
196 - else:  
197 - four = numpy.zeros((256, 256), dtype=int) + 255  
198 - four[pixel_array[:, :, 0] == 65536] = 0  
199 - r, buf = cv2.imencode(".png", numpy.dstack((pixel_array, four)))  
200 - image_out = buf.tobytes()  
201 - return image_out  
202 -  
203 - def get_data(self,zoo,image_server,image,extent,bands):  
204 -  
205 - if image_server.__eq__("本地服务器"):  
206 - data = self.get_local_data(image, extent, bands)  
207 - else:  
208 - ser = image_server  
209 - if zoo.exists("/rpc/{}".format(ser)):  
210 - data = self.get_remote_data(image, extent)  
211 - else:  
212 - data = numpy.zeros((256, 256, 3), dtype=int) + 65536  
213 - return data  
214 -  
215 -  
216 - def get_remote_data(self,image,extent,bands):  
217 - '''  
218 - 通过RPC获取远程数据  
219 - :param image:  
220 - :param extent:  
221 - :param bands:  
222 - :return:  
223 - '''  
224 -  
225 - transport = TSocket.TSocket(image.host, image.port)  
226 - transport = TTransport.TBufferedTransport(transport)  
227 - protocol = TBinaryProtocol.TBinaryProtocol(transport)  
228 - client = ImageDataService.Client(protocol)  
229 - transport.open()  
230 - t1 = time.time()  
231 - data = client.getData(image.path, extent, json.loads(image.extent), bands,256,256)  
232 - transport.close()  
233 - current_app.logger.info("time {}".format(time.time() - t1))  
234 -  
235 - data = gzip.decompress(data)  
236 - data = numpy.frombuffer(data, dtype=int)  
237 - data = data.reshape((256, 256, 3))  
238 -  
239 - return data  
240 -  
241 - def get_local_data(self,image,extent,bands):  
242 - '''  
243 - 获取本地数据  
244 - :param image:  
245 - :param extent:  
246 - :param bands:  
247 - :return:  
248 - '''  
249 - pixel_array = numpy.zeros((256, 256, 3), dtype=int)  
250 - ceng = 0  
251 - img: Dataset = gdal.Open(image.path, 0)  
252 - t1 = time.time()  
253 - for band in bands:  
254 -  
255 - # 自决定金字塔等级  
256 - xysize = [img.RasterXSize, img.RasterYSize]  
257 - origin_extent = image.extent  
258 - band_data: Band = img.GetRasterBand(band)  
259 - max_level = band_data.GetOverviewCount()  
260 -  
261 - # 超出空间范围  
262 - if extent[2] < origin_extent[0] or extent[0] > origin_extent[2] or extent[1] > origin_extent[  
263 - 3] or extent[3] < origin_extent[1]:  
264 - empty = numpy.zeros((256, 256), dtype=int) + 65536  
265 - # 空间范围相交  
266 - else:  
267 - image_level = self.determine_level(xysize, origin_extent, extent, max_level)  
268 -  
269 - if image_level == -1:  
270 - overview = band_data  
271 - else:  
272 - try:  
273 - overview: Band = band_data.GetOverview(image_level)  
274 - except:  
275 - raise Exception("该影像不存在该级别的金字塔数据!")  
276 - ox = overview.XSize  
277 - oy = overview.YSize  
278 -  
279 - # 网格大小  
280 - grid_x = (origin_extent[2] - origin_extent[0]) / (ox * 1.0)  
281 - grid_y = (origin_extent[3] - origin_extent[1]) / (oy * 1.0)  
282 -  
283 - # 完全在影像范围内  
284 - if extent[0] > origin_extent[0] and extent[1] > origin_extent[1] and extent[2] < \  
285 - origin_extent[2] and extent[3] < origin_extent[3]:  
286 -  
287 - # 网格偏移量  
288 - off_x = math.floor((extent[0] - origin_extent[0]) / grid_x)  
289 - off_y = math.floor((origin_extent[3] - extent[3]) / grid_y)  
290 -  
291 - # 截取后网格个数  
292 - x_g = math.ceil((extent[2] - extent[0]) / grid_x)  
293 -  
294 - y_g = math.ceil((extent[3] - extent[1]) / grid_y)  
295 -  
296 - empty = overview.ReadAsArray(off_x, off_y, x_g, y_g, 256, 256)  
297 -  
298 -  
299 - # 部分相交  
300 - else:  
301 -  
302 - inter_extent = [0, 0, 0, 0]  
303 - inter_extent[0] = origin_extent[0] if origin_extent[0] > extent[0] else extent[0]  
304 - inter_extent[1] = origin_extent[1] if origin_extent[1] > extent[1] else extent[1]  
305 - inter_extent[2] = origin_extent[2] if origin_extent[2] < extent[2] else extent[2]  
306 - inter_extent[3] = origin_extent[3] if origin_extent[3] < extent[3] else extent[3]  
307 -  
308 - # 网格偏移量  
309 - off_x = math.floor((inter_extent[0] - origin_extent[0]) / grid_x)  
310 - off_y = math.floor((origin_extent[3] - inter_extent[3]) / grid_y)  
311 -  
312 - # 截取后网格个数  
313 - x_g = math.floor((inter_extent[2] - inter_extent[0]) / grid_x)  
314 - y_g = math.floor((inter_extent[3] - inter_extent[1]) / grid_y)  
315 -  
316 - # 相对于出图的偏移量  
317 -  
318 - # 出图的网格大小  
319 - out_grid_x = (extent[2] - extent[0]) / (256 * 1.0)  
320 - out_grid_y = (extent[3] - extent[1]) / (256 * 1.0)  
321 -  
322 - out_off_x = int(math.ceil((inter_extent[0] - extent[0]) / out_grid_x))  
323 - out_off_y = int(math.ceil((extent[3] - inter_extent[3]) / out_grid_y))  
324 -  
325 - out_x_g = int(math.floor((inter_extent[2] - inter_extent[0]) / out_grid_x))  
326 - out_y_g = int(math.floor((inter_extent[3] - inter_extent[1]) / out_grid_y))  
327 -  
328 - # 相交部分在出图的哪个位置  
329 -  
330 - overview_raster: ndarray = overview.ReadAsArray(off_x, off_y, x_g, y_g, out_x_g,  
331 - out_y_g)  
332 -  
333 - dat = numpy.zeros((256, 256), dtype=int) + 65536  
334 - dat[out_off_y:out_off_y + out_y_g, out_off_x:out_off_x + out_x_g] = overview_raster  
335 -  
336 - empty = dat  
337 -  
338 - pixel_array[:, :, ceng] = empty  
339 - ceng += 1  
340 - return pixel_array  
341 -  
342 - def determin_intersect(self,extent1,extent2):  
343 - g1 = GeometryAdapter.envelop_2_polygon(extent1)  
344 - g2 = GeometryAdapter.envelop_2_polygon(extent2)  
345 - return g1.Intersect(g2)  
346 -  
347 - api_doc = {  
348 - "tags": ["影像接口"],  
349 - "parameters": [  
350 - {"name": "guid",  
351 - "in": "formData",  
352 - "type": "string"},  
353 - {"name": "tilematrix",  
354 - "in": "formData",  
355 - "type": "string"},  
356 - {"name": "tilerow",  
357 - "in": "formData",  
358 - "type": "string"},  
359 - {"name": "tilecol",  
360 - "in": "formData",  
361 - "type": "string"},  
362 - {"name": "format",  
363 - "in": "formData",  
364 - "type": "string"},  
365 - {"name": "quality",  
366 - "in": "formData",  
367 - "type": "string"}  
368 -  
369 - ],  
370 - "responses": {  
371 - 200: {  
372 - "schema": {  
373 - "properties": {  
374 - }  
375 - }  
376 - }  
377 - }  
378 - }  
379 -  
380 -class MyThread(Thread):  
381 - def __init__(self,func,args=()):  
382 - super(MyThread,self).__init__()  
383 - self.func = func  
384 - self.args = args  
385 - def run(self):  
386 - self.result = self.func(*self.args)  
387 - def get_result(self):  
388 - try:  
389 - return self.result  
390 - except Exception:  
391 - return None  
392 -  
393 -  
394 -  
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
@@ -5,14 +5,11 @@ @@ -5,14 +5,11 @@
5 5
6 6
7 from app.util.component.ApiTemplate import ApiTemplate 7 from app.util.component.ApiTemplate import ApiTemplate
8 -from thrift.transport import TSocket  
9 -from thrift.transport import TTransport  
10 -from thrift.protocol import TBinaryProtocol 8 +
11 import json 9 import json
12 -from app.modules.image.ImageDataService import ImageDataService  
13 from app.util.component.FileProcess import FileProcess 10 from app.util.component.FileProcess import FileProcess
14 import datetime 11 import datetime
15 -from app.util.component.ThriftConnect import ThriftConnect 12 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
16 import os 13 import os
17 class Api(ApiTemplate): 14 class Api(ApiTemplate):
18 15
  1 +<?xml version="1.0" encoding="utf-8" ?><CacheInfo xsi:type='typens:CacheInfo' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.1'><TileCacheInfo xsi:type='typens:TileCacheInfo'><SpatialReference xsi:type='typens:GeographicCoordinateSystem'><WKT>GEOGCS[&quot;GCS_China_Geodetic_Coordinate_System_2000&quot;,DATUM[&quot;D_China_2000&quot;,SPHEROID[&quot;CGCS2000&quot;,6378137.0,298.257222101]],PRIMEM[&quot;Greenwich&quot;,0.0],UNIT[&quot;Degree&quot;,0.0174532925199433],METADATA[&quot;China&quot;,73.66,7.16,134.85,53.59,0.0,0.0174532925199433,0.0,1067],AUTHORITY[&quot;EPSG&quot;,4490]]</WKT><XOrigin>-400</XOrigin><YOrigin>-400</YOrigin><XYScale>11258999068426.24</XYScale><ZOrigin>-100000</ZOrigin><ZScale>10000</ZScale><MOrigin>-100000</MOrigin><MScale>10000</MScale><XYTolerance>8.9831528411952133e-009</XYTolerance><ZTolerance>0.001</ZTolerance><MTolerance>0.001</MTolerance><HighPrecision>true</HighPrecision><LeftLongitude>-180</LeftLongitude><WKID>4490</WKID><LatestWKID>4490</LatestWKID></SpatialReference><TileOrigin xsi:type='typens:PointN'><X>-400</X><Y>399.99999999999977</Y></TileOrigin><TileCols>256</TileCols><TileRows>256</TileRows><DPI>96</DPI><PreciseDPI>96</PreciseDPI><LODInfos xsi:type='typens:ArrayOfLODInfo'><LODInfo xsi:type='typens:LODInfo'><LevelID>0</LevelID><Scale>1154287.473</Scale><Resolution>0.0027465820315218724</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>1</LevelID><Scale>577143.73640000005</Scale><Resolution>0.0013732910155229902</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>2</LevelID><Scale>288571.86820000003</Scale><Resolution>0.00068664550776149512</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>3</LevelID><Scale>144285.93410000001</Scale><Resolution>0.00034332275388074756</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>4</LevelID><Scale>72142.967059999995</Scale><Resolution>0.00017166137696416836</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>5</LevelID><Scale>36071.483529999998</Scale><Resolution>8.5830688482084179e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>6</LevelID><Scale>18035.741760000001</Scale><Resolution>4.2915344229144793e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>7</LevelID><Scale>9017.8708819999993</Scale><Resolution>2.1457672119331316e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>8</LevelID><Scale>4508.9354409999996</Scale><Resolution>1.0728836059665658e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>9</LevelID><Scale>2254.4677200000001</Scale><Resolution>5.3644180286430992e-006</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>10</LevelID><Scale>1127.23386</Scale><Resolution>2.6822090143215496e-006</Resolution></LODInfo></LODInfos></TileCacheInfo><TileImageInfo xsi:type='typens:TileImageInfo'><CacheTileFormat>PNG32</CacheTileFormat><CompressionQuality>0</CompressionQuality><Antialiasing>false</Antialiasing></TileImageInfo><CacheStorageInfo xsi:type='typens:CacheStorageInfo'><StorageFormat>esriMapCacheStorageModeCompact</StorageFormat><PacketSize>128</PacketSize></CacheStorageInfo></CacheInfo>
@@ -7,7 +7,7 @@ @@ -7,7 +7,7 @@
7 from app.util.component.ApiTemplate import ApiTemplate 7 from app.util.component.ApiTemplate import ApiTemplate
8 from app.util.component.ModelVisitor import ModelVisitor 8 from app.util.component.ModelVisitor import ModelVisitor
9 9
10 -from app.modules.image.models import Image 10 +from app.modules.service.image.models import Image
11 from sqlalchemy import or_,and_ 11 from sqlalchemy import or_,and_
12 class Api(ApiTemplate): 12 class Api(ApiTemplate):
13 13
@@ -7,8 +7,7 @@ @@ -7,8 +7,7 @@
7 from osgeo import gdal,ogr,osr 7 from osgeo import gdal,ogr,osr
8 from osgeo.gdal import Dataset,Band 8 from osgeo.gdal import Dataset,Band
9 from app.util.component.ApiTemplate import ApiTemplate 9 from app.util.component.ApiTemplate import ApiTemplate
10 -from app.util.component.ThriftConnect import ThriftConnect  
11 - 10 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
12 import json 11 import json
13 from .models import Image 12 from .models import Image
14 import datetime 13 import datetime
@@ -6,7 +6,7 @@ @@ -6,7 +6,7 @@
6 6
7 from app.util.component.ApiTemplate import ApiTemplate 7 from app.util.component.ApiTemplate import ApiTemplate
8 from app.util.component.ModelVisitor import ModelVisitor 8 from app.util.component.ModelVisitor import ModelVisitor
9 -from app.modules.image.models import ImageService 9 +from app.modules.service.image.models import ImageService
10 from sqlalchemy import or_ 10 from sqlalchemy import or_
11 11
12 class Api(ApiTemplate): 12 class Api(ApiTemplate):
@@ -3,7 +3,7 @@ @@ -3,7 +3,7 @@
3 #createtime: 2021/7/19 3 #createtime: 2021/7/19
4 #email: nheweijun@sina.com 4 #email: nheweijun@sina.com
5 5
6 -from app.modules.image.models import ImageService,Image 6 +from app.modules.service.image.models import ImageService,Image
7 from app.models import db 7 from app.models import db
8 from app.util.component.ApiTemplate import ApiTemplate 8 from app.util.component.ApiTemplate import ApiTemplate
9 import uuid 9 import uuid
@@ -16,7 +16,7 @@ from PIL import Image @@ -16,7 +16,7 @@ from PIL import Image
16 16
17 import time 17 import time
18 import cv2 18 import cv2
19 -from app.modules.image.models import ImageService,Image 19 +from app.modules.service.image.models import ImageService,Image
20 from app.models import db 20 from app.models import db
21 from app.util.component.ApiTemplate import ApiTemplate 21 from app.util.component.ApiTemplate import ApiTemplate
22 import uuid 22 import uuid
@@ -30,7 +30,7 @@ from kazoo.client import KazooClient @@ -30,7 +30,7 @@ from kazoo.client import KazooClient
30 from app import GLOBAL_DIC 30 from app import GLOBAL_DIC
31 from threading import Thread 31 from threading import Thread
32 32
33 -from app.util.component.ThriftConnect import ThriftConnect 33 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
34 from flask import current_app 34 from flask import current_app
35 import gzip 35 import gzip
36 import random 36 import random
@@ -13,10 +13,10 @@ from flask import Response @@ -13,10 +13,10 @@ from flask import Response
13 import random 13 import random
14 import time 14 import time
15 import cv2 15 import cv2
16 -from app.modules.image.models import ImageService 16 +from app.modules.service.image.models import ImageService
17 from app.util.component.ApiTemplate import ApiTemplate 17 from app.util.component.ApiTemplate import ApiTemplate
18 from app.util.component.GeometryAdapter import GeometryAdapter 18 from app.util.component.GeometryAdapter import GeometryAdapter
19 -from app.util.component.ThriftConnect import ThriftConnect 19 +from app.modules.service.image.util.ThriftConnect import ThriftConnect
20 from app.util.component.ParameterUtil import ParameterUtil 20 from app.util.component.ParameterUtil import ParameterUtil
21 import json 21 import json
22 from kazoo.client import KazooClient 22 from kazoo.client import KazooClient
@@ -6,7 +6,7 @@ @@ -6,7 +6,7 @@
6 from thrift.transport import TSocket 6 from thrift.transport import TSocket
7 from thrift.transport import TTransport 7 from thrift.transport import TTransport
8 from thrift.protocol import TBinaryProtocol 8 from thrift.protocol import TBinaryProtocol
9 -from app.modules.image.ImageDataService import ImageDataService 9 +from app.modules.service.image.ImageDataService import ImageDataService
10 10
11 class ThriftConnect: 11 class ThriftConnect:
12 ''' 12 '''
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
  1 +# coding=utf-8
  2 +#author: 4N
  3 +#createtime: 2021/9/14
  4 +#email: nheweijun@sina.com
注册登录 后发表评论