正在显示
87 个修改的文件
包含
179 行增加
和
547 行删除
... | ... | @@ -3,7 +3,6 @@ import decimal |
3 | 3 | from flask import Flask as _Flask |
4 | 4 | from flask.json import JSONEncoder as _JSONEncoder |
5 | 5 | from flask_cors import CORS |
6 | -import time | |
7 | 6 | |
8 | 7 | from sqlalchemy.sql.expression import true |
9 | 8 | import configure |
... | ... | @@ -12,24 +11,14 @@ from app.util import find_class |
12 | 11 | from app.models import db, Table, InsertingLayerName, Database, DES, Task |
13 | 12 | from app.modules.auth.oauth2 import config_oauth, myCodeIDToken |
14 | 13 | from flasgger import Swagger |
15 | -# from rtree import index | |
16 | 14 | import logging |
17 | -from sqlalchemy.orm import Session | |
18 | -import multiprocessing | |
19 | 15 | from app.util.component.EntryData import EntryData |
20 | 16 | from app.util.component.EntryDataVacuate import EntryDataVacuate |
21 | -import json | |
22 | 17 | import threading |
23 | -import traceback | |
24 | -from sqlalchemy import distinct | |
25 | -import uuid | |
26 | -from osgeo.ogr import DataSource | |
27 | -import datetime | |
28 | -from sqlalchemy import or_ | |
29 | 18 | from app.util.component.StructuredPrint import StructurePrint |
30 | 19 | from app.util.component.PGUtil import PGUtil |
31 | 20 | import os |
32 | - | |
21 | +from app.modules.data.io.data_entry_center import data_entry_center | |
33 | 22 | |
34 | 23 | class JSONEncoder(_JSONEncoder): |
35 | 24 | """ |
... | ... | @@ -105,7 +94,6 @@ def create_app(): |
105 | 94 | app.register_blueprint(api.bp) |
106 | 95 | |
107 | 96 | # 入库监测线程 |
108 | - | |
109 | 97 | @app.before_first_request |
110 | 98 | def data_entry_process(): |
111 | 99 | StructurePrint.print("start listen") |
... | ... | @@ -120,99 +108,3 @@ def create_app(): |
120 | 108 | |
121 | 109 | return app |
122 | 110 | |
123 | - | |
124 | -def data_entry_center(): | |
125 | - running_dict = {} | |
126 | - sys_session: Session = PGUtil.get_db_session( | |
127 | - configure.SQLALCHEMY_DATABASE_URI) | |
128 | - | |
129 | - while True: | |
130 | - | |
131 | - try: | |
132 | - time.sleep(3) | |
133 | - | |
134 | - # 已经结束的进程 从监测中删除 | |
135 | - remove_process = [] | |
136 | - | |
137 | - # structured_print(running_dict.__len__().__str__()) | |
138 | - | |
139 | - for process, layer_names in running_dict.items(): | |
140 | - if not process.is_alive(): | |
141 | - for l in layer_names: | |
142 | - inserted = sys_session.query( | |
143 | - InsertingLayerName).filter_by(name=l).one_or_none() | |
144 | - if inserted: | |
145 | - sys_session.delete(inserted) | |
146 | - sys_session.commit() | |
147 | - remove_process.append(process) | |
148 | - for process in remove_process: | |
149 | - running_dict.pop(process) | |
150 | - | |
151 | - # StructurePrint.print("listening...") | |
152 | - | |
153 | - # 入库进程少于阈值,开启入库进程 | |
154 | - | |
155 | - inter_size = sys_session.query( | |
156 | - distinct(InsertingLayerName.task_guid)).count() | |
157 | - | |
158 | - if inter_size < configure.entry_data_thread: | |
159 | - # 锁表啊 | |
160 | - ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | |
161 | - Task.create_time).with_lockmode("update").limit(1).one_or_none() | |
162 | - if ready_task: | |
163 | - | |
164 | - try: | |
165 | - parameter = json.loads(ready_task.parameter) | |
166 | - StructurePrint.print("检测到入库任务") | |
167 | - ready_task.state = 2 | |
168 | - ready_task.process = "入库中" | |
169 | - sys_session.commit() | |
170 | - | |
171 | - metas: list = json.loads( | |
172 | - parameter.get("meta").__str__()) | |
173 | - parameter["meta"] = metas | |
174 | - | |
175 | - database = sys_session.query(Database).filter_by( | |
176 | - guid=ready_task.database_guid).one_or_none() | |
177 | - pg_ds: DataSource = PGUtil.open_pg_data_source( | |
178 | - 1, DES.decode(database.sqlalchemy_uri)) | |
179 | - | |
180 | - this_task_layer = [] | |
181 | - for meta in metas: | |
182 | - overwrite = parameter.get("overwrite", "no") | |
183 | - | |
184 | - for layer_name_origin, layer_name in meta.get("layer").items(): | |
185 | - origin_name = layer_name | |
186 | - no = 1 | |
187 | - | |
188 | - while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | |
189 | - layer_name = origin_name + "_{}".format(no) | |
190 | - no += 1 | |
191 | - | |
192 | - # 添加到正在入库的列表中 | |
193 | - iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | |
194 | - task_guid=ready_task.guid, | |
195 | - name=layer_name) | |
196 | - | |
197 | - sys_session.add(iln) | |
198 | - sys_session.commit() | |
199 | - this_task_layer.append(layer_name) | |
200 | - # 修改表名 | |
201 | - meta["layer"][layer_name_origin] = layer_name | |
202 | - | |
203 | - pg_ds.Destroy() | |
204 | - entry_data_process = multiprocessing.Process( | |
205 | - target=EntryDataVacuate().entry, args=(parameter,)) | |
206 | - entry_data_process.start() | |
207 | - running_dict[entry_data_process] = this_task_layer | |
208 | - except Exception as e: | |
209 | - sys_session.query(Task).filter_by(guid=ready_task.guid).update( | |
210 | - {"state": -1, "process": "入库失败"}) | |
211 | - sys_session.commit() | |
212 | - StructurePrint.print(e.__str__(), "error") | |
213 | - else: | |
214 | - # 解表啊 | |
215 | - sys_session.commit() | |
216 | - except Exception as e: | |
217 | - sys_session.commit() | |
218 | - StructurePrint.print(e.__str__(), "error") | |
\ No newline at end of file | ... | ... |
... | ... | @@ -34,13 +34,14 @@ class DES(): |
34 | 34 | db = SQLAlchemy() |
35 | 35 | |
36 | 36 | # 动态加载Model |
37 | -module_dir = "modules" | |
38 | -model_pkg = "models" | |
39 | 37 | current_dir = os.path.abspath(os.path.dirname(__file__)) |
40 | -for pkg in glob.glob('%s/%s/*/%s' % (current_dir,module_dir,model_pkg)) : | |
41 | - module = os.path.basename(os.path.dirname(pkg)) | |
42 | - base_name = os.path.basename(pkg).split('.')[0] | |
43 | - pkg_name = ".".join(["app.modules",module,base_name]) | |
38 | +pkgs = list(glob.glob('%s/modules/*/models' % (current_dir))) | |
39 | +pkgs.extend(list(glob.glob('%s/modules/*/*/models' % (current_dir)))) | |
40 | + | |
41 | +for pkg in pkgs : | |
42 | + pkg = os.path.normpath(pkg) | |
43 | + node_list = pkg.split(os.path.sep) | |
44 | + pkg_name = ".".join(node_list[node_list.index("app"):]) | |
44 | 45 | try: |
45 | 46 | __import__(pkg_name) |
46 | 47 | except: |
... | ... | @@ -199,21 +200,21 @@ class InsertingLayerName(db.Model): |
199 | 200 | name = Column(String(256)) |
200 | 201 | |
201 | 202 | |
202 | -# class Service(db.Model): | |
203 | -# ''' | |
204 | -# ''' | |
205 | -# __tablename__ = 'dmdms_service' | |
206 | -# guid = Column(String(256), primary_key=True) | |
207 | -# name=Column(String(256)) | |
208 | -# alias = Column(String(256)) | |
209 | -# #服务状态 | |
210 | -# state= Column(Integer) | |
211 | -# create_time = Column(DateTime) | |
212 | -# update_time = Column(DateTime) | |
213 | -# #服务描述 | |
214 | -# description = Column(Text) | |
215 | -# #服务节点 | |
216 | -# node = Column(Integer) | |
217 | -# #服务缩略图 | |
218 | -# overview = Column(Binary) | |
203 | +class Service(db.Model): | |
204 | + ''' | |
205 | + ''' | |
206 | + __tablename__ = 'dmdms_service' | |
207 | + guid = Column(String(256), primary_key=True) | |
208 | + name=Column(String(256)) | |
209 | + alias = Column(String(256)) | |
210 | + #服务状态 | |
211 | + state= Column(Integer) | |
212 | + create_time = Column(DateTime) | |
213 | + update_time = Column(DateTime) | |
214 | + #服务描述 | |
215 | + description = Column(Text) | |
216 | + #服务节点 | |
217 | + node = Column(Integer) | |
218 | + #服务缩略图 | |
219 | + overview = Column(Binary) | |
219 | 220 | ... | ... |
app/modules/data/__init__.py
0 → 100644
... | ... | @@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate |
8 | 8 | import uuid |
9 | 9 | from . import database_test |
10 | 10 | # from app.util import open_pg_data_source |
11 | -from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn,Feature,Geometry | |
11 | +from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn | |
12 | 12 | from sqlalchemy.orm import Session |
13 | 13 | from sqlalchemy import create_engine |
14 | 14 | from sqlalchemy.orm import sessionmaker |
... | ... | @@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil |
17 | 17 | from app.util.component.SQLUtil import SQLUtil |
18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
19 | 19 | from app.util.component.StructuredPrint import StructurePrint |
20 | -import configure | |
20 | + | |
21 | + | |
21 | 22 | class Api(ApiTemplate): |
22 | 23 | api_name = "注册数据库" |
23 | 24 | def process(self): | ... | ... |
... | ... | @@ -6,15 +6,13 @@ from flasgger import swag_from |
6 | 6 | from flask import Blueprint |
7 | 7 | from app.util import BlueprintApi |
8 | 8 | |
9 | -from flask import send_from_directory,current_app,send_file | |
9 | +from flask import send_from_directory | |
10 | 10 | import os |
11 | -import shutil | |
12 | 11 | from . import data_download |
13 | 12 | from . import get_meta |
14 | 13 | from . import data_entry_by_meta |
15 | 14 | from . import get_data_list |
16 | -from flask import after_this_request | |
17 | -import time | |
15 | + | |
18 | 16 | |
19 | 17 | class DataManager(BlueprintApi): |
20 | 18 | ... | ... |
app/modules/data/io/data_entry_center.py
0 → 100644
1 | +# coding=utf-8 | |
2 | +#author: 4N | |
3 | +#createtime: 2021/9/14 | |
4 | +#email: nheweijun@sina.com | |
5 | + | |
6 | +import configure | |
7 | +from app.models import InsertingLayerName, Database, DES, Task | |
8 | +from sqlalchemy.orm import Session | |
9 | +import multiprocessing | |
10 | +from app.util.component.EntryDataVacuate import EntryDataVacuate | |
11 | +import json | |
12 | +from sqlalchemy import distinct | |
13 | +import uuid | |
14 | +from osgeo.ogr import DataSource | |
15 | +from app.util.component.StructuredPrint import StructurePrint | |
16 | +from app.util.component.PGUtil import PGUtil | |
17 | +import time | |
18 | + | |
19 | +def data_entry_center(): | |
20 | + running_dict = {} | |
21 | + sys_session: Session = PGUtil.get_db_session( | |
22 | + configure.SQLALCHEMY_DATABASE_URI) | |
23 | + | |
24 | + while True: | |
25 | + | |
26 | + try: | |
27 | + time.sleep(3) | |
28 | + | |
29 | + # 已经结束的进程 从监测中删除 | |
30 | + remove_process = [] | |
31 | + | |
32 | + # structured_print(running_dict.__len__().__str__()) | |
33 | + | |
34 | + for process, layer_names in running_dict.items(): | |
35 | + if not process.is_alive(): | |
36 | + for l in layer_names: | |
37 | + inserted = sys_session.query( | |
38 | + InsertingLayerName).filter_by(name=l).one_or_none() | |
39 | + if inserted: | |
40 | + sys_session.delete(inserted) | |
41 | + sys_session.commit() | |
42 | + remove_process.append(process) | |
43 | + for process in remove_process: | |
44 | + running_dict.pop(process) | |
45 | + | |
46 | + # StructurePrint.print("listening...") | |
47 | + | |
48 | + # 入库进程少于阈值,开启入库进程 | |
49 | + | |
50 | + inter_size = sys_session.query( | |
51 | + distinct(InsertingLayerName.task_guid)).count() | |
52 | + | |
53 | + if inter_size < configure.entry_data_thread: | |
54 | + # 锁表啊 | |
55 | + ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | |
56 | + Task.create_time).with_lockmode("update").limit(1).one_or_none() | |
57 | + if ready_task: | |
58 | + | |
59 | + try: | |
60 | + parameter = json.loads(ready_task.parameter) | |
61 | + StructurePrint.print("检测到入库任务") | |
62 | + ready_task.state = 2 | |
63 | + ready_task.process = "入库中" | |
64 | + sys_session.commit() | |
65 | + | |
66 | + metas: list = json.loads( | |
67 | + parameter.get("meta").__str__()) | |
68 | + parameter["meta"] = metas | |
69 | + | |
70 | + database = sys_session.query(Database).filter_by( | |
71 | + guid=ready_task.database_guid).one_or_none() | |
72 | + pg_ds: DataSource = PGUtil.open_pg_data_source( | |
73 | + 1, DES.decode(database.sqlalchemy_uri)) | |
74 | + | |
75 | + this_task_layer = [] | |
76 | + for meta in metas: | |
77 | + overwrite = parameter.get("overwrite", "no") | |
78 | + | |
79 | + for layer_name_origin, layer_name in meta.get("layer").items(): | |
80 | + origin_name = layer_name | |
81 | + no = 1 | |
82 | + | |
83 | + while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | |
84 | + layer_name = origin_name + "_{}".format(no) | |
85 | + no += 1 | |
86 | + | |
87 | + # 添加到正在入库的列表中 | |
88 | + iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | |
89 | + task_guid=ready_task.guid, | |
90 | + name=layer_name) | |
91 | + | |
92 | + sys_session.add(iln) | |
93 | + sys_session.commit() | |
94 | + this_task_layer.append(layer_name) | |
95 | + # 修改表名 | |
96 | + meta["layer"][layer_name_origin] = layer_name | |
97 | + | |
98 | + pg_ds.Destroy() | |
99 | + entry_data_process = multiprocessing.Process( | |
100 | + target=EntryDataVacuate().entry, args=(parameter,)) | |
101 | + entry_data_process.start() | |
102 | + running_dict[entry_data_process] = this_task_layer | |
103 | + except Exception as e: | |
104 | + sys_session.query(Task).filter_by(guid=ready_task.guid).update( | |
105 | + {"state": -1, "process": "入库失败"}) | |
106 | + sys_session.commit() | |
107 | + StructurePrint.print(e.__str__(), "error") | |
108 | + else: | |
109 | + # 解表啊 | |
110 | + sys_session.commit() | |
111 | + except Exception as e: | |
112 | + sys_session.commit() | |
113 | + StructurePrint.print(e.__str__(), "error") | |
\ No newline at end of file | ... | ... |
app/modules/image/image_tilebac.py
deleted
100644 → 0
1 | -# coding=utf-8 | |
2 | -#author: 4N | |
3 | -#createtime: 2021/3/24 | |
4 | -#email: nheweijun@sina.com | |
5 | - | |
6 | -from app.util import * | |
7 | -import traceback | |
8 | -from osgeo import gdal | |
9 | -from osgeo.gdal import * | |
10 | -from numpy import ndarray | |
11 | -import numpy | |
12 | -from flask import Response | |
13 | -import io | |
14 | -import os | |
15 | -from PIL import Image | |
16 | - | |
17 | -import time | |
18 | -import cv2 | |
19 | -from app.modules.image.models import ImageService,Image | |
20 | -from app.models import db | |
21 | -from app.util.component.ApiTemplate import ApiTemplate | |
22 | -import uuid | |
23 | -from app.util.component.SliceScheme import SliceScheme | |
24 | -from app.util.component.FileProcess import FileProcess | |
25 | -from app.util.component.ParameterUtil import ParameterUtil | |
26 | -from app.util.component.GeometryAdapter import GeometryAdapter | |
27 | -import os | |
28 | -import json | |
29 | -from kazoo.client import KazooClient | |
30 | -from app import GLOBAL_DIC | |
31 | -from threading import Thread | |
32 | -from thrift.transport import TSocket | |
33 | -from thrift.transport import TTransport | |
34 | -from thrift.protocol import TBinaryProtocol | |
35 | -from .ImageDataService import ImageDataService | |
36 | -from flask import current_app | |
37 | -import gzip | |
38 | -import random | |
39 | -class Api(ApiTemplate): | |
40 | - | |
41 | - api_name = "切片" | |
42 | - | |
43 | - def __init__(self,guid,level, row, col): | |
44 | - super().__init__() | |
45 | - self.guid = guid | |
46 | - self.level = level | |
47 | - self.row = row | |
48 | - self.col = col | |
49 | - | |
50 | - def process(self): | |
51 | - | |
52 | - result = {} | |
53 | - parameter: dict = self.para | |
54 | - | |
55 | - try: | |
56 | - if parameter.get("guid"): | |
57 | - self.guid = parameter.get("guid") | |
58 | - | |
59 | - image_service = ImageService.query.filter_by(guid = self.guid).one_or_none() | |
60 | - images = image_service.images.all() | |
61 | - | |
62 | - zoo = GLOBAL_DIC.get("zookeeper") | |
63 | - if zoo is None: | |
64 | - zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100) | |
65 | - zoo.start() | |
66 | - GLOBAL_DIC["zookeeper"] = zoo | |
67 | - else : | |
68 | - if not zoo.connected: | |
69 | - zoo.start() | |
70 | - | |
71 | - bands = [1, 2, 3] | |
72 | - | |
73 | - | |
74 | - # 转换参数 | |
75 | - parameter = ParameterUtil.to_lower(parameter) | |
76 | - | |
77 | - | |
78 | - if parameter.get("tilematrix"): | |
79 | - if parameter.get("tilematrix").__contains__(":"): | |
80 | - self.level = int(parameter.get("tilematrix").split(":")[-1]) | |
81 | - else: | |
82 | - self.level = int(parameter.get("tilematrix")) | |
83 | - if parameter.get("tilerow"): | |
84 | - self.row = int(parameter.get("tilerow")) | |
85 | - if parameter.get("tilecol"): | |
86 | - self.col = int(parameter.get("tilecol")) | |
87 | - | |
88 | - image_type = parameter.get("format") if parameter.get("format") else "image/png" | |
89 | - quality = int(parameter.get("quality")) if parameter.get("quality") else 30 | |
90 | - slice_para = json.loads(image_service.slice_scheme) | |
91 | - extent = SliceScheme.get_polygon(slice_para, self.level, self.row, self.col) | |
92 | - | |
93 | - | |
94 | - | |
95 | - # 多线程获取分布式数据 | |
96 | - | |
97 | - intersect_image = [im for im in images if self.determin_intersect(json.loads(im.extent),extent)] | |
98 | - | |
99 | - pixel_array = numpy.zeros((256, 256,3), dtype=int) | |
100 | - | |
101 | - if len(intersect_image) > 1: | |
102 | - | |
103 | - # 结果矩阵 | |
104 | - empty_list = [numpy.zeros((256,256), dtype=int) + 65536, | |
105 | - numpy.zeros((256,256), dtype=int) + 65536, | |
106 | - numpy.zeros((256,256), dtype=int) + 65536] | |
107 | - | |
108 | - pixel_array = numpy.zeros((256,256,3), dtype=int) | |
109 | - thread_list = [] | |
110 | - | |
111 | - for image in intersect_image: | |
112 | - #该影像的服务器,随机选取一个 | |
113 | - image_servers = image.server.split(",") | |
114 | - indx = int(random.random()*len(image_servers)) | |
115 | - image_server = image_servers[indx] | |
116 | - | |
117 | - thread: MyThread = MyThread(self.get_data, args=(zoo,image_server,image,extent,bands)) | |
118 | - thread.start() | |
119 | - thread_list.append(thread) | |
120 | - | |
121 | - | |
122 | - for thread in thread_list: | |
123 | - thread.join() | |
124 | - data = thread.get_result() | |
125 | - | |
126 | - # 掩膜在中央接口生成,合图 | |
127 | - mask = numpy.zeros((256,256), dtype=int) | |
128 | - mask2 = numpy.zeros((256,256), dtype=int) | |
129 | - jizhun = data[:, :, 0] | |
130 | - mask[jizhun == 65536] = 1 | |
131 | - mask[jizhun != 65536] = 0 | |
132 | - mask2[jizhun == 65536] = 0 | |
133 | - mask2[jizhun != 65536] = 1 | |
134 | - # 掩膜计算 | |
135 | - for i, d in enumerate(empty_list): | |
136 | - empty_list[i] = empty_list[i] * mask + data[:, :, i] * mask2 | |
137 | - for ii in [0, 1, 2]: | |
138 | - # opencv 颜色排序为GBR | |
139 | - pixel_array[:, :, 2 - ii] = empty_list[ii] | |
140 | - | |
141 | - elif len(intersect_image) == 1: | |
142 | - | |
143 | - # 该影像的服务器,随机选取一个 | |
144 | - image = intersect_image[0] | |
145 | - image_servers = image.server.split(",") | |
146 | - indx = int(random.random() * len(image_servers)) | |
147 | - image_server = image_servers[indx] | |
148 | - pixel_array_t = self.get_data(zoo, image_server, image, extent, bands) | |
149 | - pixel_array = numpy.zeros((256,256, 3), dtype=int) | |
150 | - for ii in [0, 1, 2]: | |
151 | - # opencv 颜色排序为GBR | |
152 | - pixel_array[:, :, 2 - ii] = pixel_array_t[:, :, ii] | |
153 | - | |
154 | - else: | |
155 | - # 结果矩阵 | |
156 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) + 65536 | |
157 | - | |
158 | - #将图片生成在内存中,然后直接返回response | |
159 | - im_data = self.create_by_opencv(image_type, pixel_array, quality) | |
160 | - | |
161 | - return Response(im_data, mimetype=image_type.lower()) | |
162 | - | |
163 | - except Exception as e: | |
164 | - print(traceback.format_exc()) | |
165 | - result["state"] = -1 | |
166 | - result["message"] = e.__str__() | |
167 | - return result | |
168 | - | |
169 | - | |
170 | - def determine_level(self,xysize,origin_extent,extent,max_level): | |
171 | - ''' | |
172 | - 根据范围判断调用金字塔的哪一层 | |
173 | - :param xysize: | |
174 | - :param origin_extent: | |
175 | - :param extent: | |
176 | - :param max_level: | |
177 | - :return: | |
178 | - ''' | |
179 | - x = xysize[0] | |
180 | - y = xysize[1] | |
181 | - level = -1 | |
182 | - pixel = x*y * (((extent[2]-extent[0])*(extent[3]-extent[1]))/((origin_extent[2]-origin_extent[0])*(origin_extent[3]-origin_extent[1]))) | |
183 | - while pixel>100000 and level<max_level-1: | |
184 | - level+=1 | |
185 | - x=x/2 | |
186 | - y=y/2 | |
187 | - pixel = x * y * (((extent[2] - extent[0]) * (extent[3] - extent[1])) / ( | |
188 | - (origin_extent[2] - origin_extent[0]) * (origin_extent[3] - origin_extent[1]))) | |
189 | - return level | |
190 | - | |
191 | - def create_by_opencv(self,image_type, pixel_array, quality): | |
192 | - | |
193 | - if image_type.__eq__("image/jpeg") or image_type.__eq__("image/jpg"): | |
194 | - r, buf = cv2.imencode(".jpg", pixel_array, [cv2.IMWRITE_JPEG_QUALITY, quality]) | |
195 | - image_out = buf.tobytes() | |
196 | - else: | |
197 | - four = numpy.zeros((256, 256), dtype=int) + 255 | |
198 | - four[pixel_array[:, :, 0] == 65536] = 0 | |
199 | - r, buf = cv2.imencode(".png", numpy.dstack((pixel_array, four))) | |
200 | - image_out = buf.tobytes() | |
201 | - return image_out | |
202 | - | |
203 | - def get_data(self,zoo,image_server,image,extent,bands): | |
204 | - | |
205 | - if image_server.__eq__("本地服务器"): | |
206 | - data = self.get_local_data(image, extent, bands) | |
207 | - else: | |
208 | - ser = image_server | |
209 | - if zoo.exists("/rpc/{}".format(ser)): | |
210 | - data = self.get_remote_data(image, extent) | |
211 | - else: | |
212 | - data = numpy.zeros((256, 256, 3), dtype=int) + 65536 | |
213 | - return data | |
214 | - | |
215 | - | |
216 | - def get_remote_data(self,image,extent,bands): | |
217 | - ''' | |
218 | - 通过RPC获取远程数据 | |
219 | - :param image: | |
220 | - :param extent: | |
221 | - :param bands: | |
222 | - :return: | |
223 | - ''' | |
224 | - | |
225 | - transport = TSocket.TSocket(image.host, image.port) | |
226 | - transport = TTransport.TBufferedTransport(transport) | |
227 | - protocol = TBinaryProtocol.TBinaryProtocol(transport) | |
228 | - client = ImageDataService.Client(protocol) | |
229 | - transport.open() | |
230 | - t1 = time.time() | |
231 | - data = client.getData(image.path, extent, json.loads(image.extent), bands,256,256) | |
232 | - transport.close() | |
233 | - current_app.logger.info("time {}".format(time.time() - t1)) | |
234 | - | |
235 | - data = gzip.decompress(data) | |
236 | - data = numpy.frombuffer(data, dtype=int) | |
237 | - data = data.reshape((256, 256, 3)) | |
238 | - | |
239 | - return data | |
240 | - | |
241 | - def get_local_data(self,image,extent,bands): | |
242 | - ''' | |
243 | - 获取本地数据 | |
244 | - :param image: | |
245 | - :param extent: | |
246 | - :param bands: | |
247 | - :return: | |
248 | - ''' | |
249 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) | |
250 | - ceng = 0 | |
251 | - img: Dataset = gdal.Open(image.path, 0) | |
252 | - t1 = time.time() | |
253 | - for band in bands: | |
254 | - | |
255 | - # 自决定金字塔等级 | |
256 | - xysize = [img.RasterXSize, img.RasterYSize] | |
257 | - origin_extent = image.extent | |
258 | - band_data: Band = img.GetRasterBand(band) | |
259 | - max_level = band_data.GetOverviewCount() | |
260 | - | |
261 | - # 超出空间范围 | |
262 | - if extent[2] < origin_extent[0] or extent[0] > origin_extent[2] or extent[1] > origin_extent[ | |
263 | - 3] or extent[3] < origin_extent[1]: | |
264 | - empty = numpy.zeros((256, 256), dtype=int) + 65536 | |
265 | - # 空间范围相交 | |
266 | - else: | |
267 | - image_level = self.determine_level(xysize, origin_extent, extent, max_level) | |
268 | - | |
269 | - if image_level == -1: | |
270 | - overview = band_data | |
271 | - else: | |
272 | - try: | |
273 | - overview: Band = band_data.GetOverview(image_level) | |
274 | - except: | |
275 | - raise Exception("该影像不存在该级别的金字塔数据!") | |
276 | - ox = overview.XSize | |
277 | - oy = overview.YSize | |
278 | - | |
279 | - # 网格大小 | |
280 | - grid_x = (origin_extent[2] - origin_extent[0]) / (ox * 1.0) | |
281 | - grid_y = (origin_extent[3] - origin_extent[1]) / (oy * 1.0) | |
282 | - | |
283 | - # 完全在影像范围内 | |
284 | - if extent[0] > origin_extent[0] and extent[1] > origin_extent[1] and extent[2] < \ | |
285 | - origin_extent[2] and extent[3] < origin_extent[3]: | |
286 | - | |
287 | - # 网格偏移量 | |
288 | - off_x = math.floor((extent[0] - origin_extent[0]) / grid_x) | |
289 | - off_y = math.floor((origin_extent[3] - extent[3]) / grid_y) | |
290 | - | |
291 | - # 截取后网格个数 | |
292 | - x_g = math.ceil((extent[2] - extent[0]) / grid_x) | |
293 | - | |
294 | - y_g = math.ceil((extent[3] - extent[1]) / grid_y) | |
295 | - | |
296 | - empty = overview.ReadAsArray(off_x, off_y, x_g, y_g, 256, 256) | |
297 | - | |
298 | - | |
299 | - # 部分相交 | |
300 | - else: | |
301 | - | |
302 | - inter_extent = [0, 0, 0, 0] | |
303 | - inter_extent[0] = origin_extent[0] if origin_extent[0] > extent[0] else extent[0] | |
304 | - inter_extent[1] = origin_extent[1] if origin_extent[1] > extent[1] else extent[1] | |
305 | - inter_extent[2] = origin_extent[2] if origin_extent[2] < extent[2] else extent[2] | |
306 | - inter_extent[3] = origin_extent[3] if origin_extent[3] < extent[3] else extent[3] | |
307 | - | |
308 | - # 网格偏移量 | |
309 | - off_x = math.floor((inter_extent[0] - origin_extent[0]) / grid_x) | |
310 | - off_y = math.floor((origin_extent[3] - inter_extent[3]) / grid_y) | |
311 | - | |
312 | - # 截取后网格个数 | |
313 | - x_g = math.floor((inter_extent[2] - inter_extent[0]) / grid_x) | |
314 | - y_g = math.floor((inter_extent[3] - inter_extent[1]) / grid_y) | |
315 | - | |
316 | - # 相对于出图的偏移量 | |
317 | - | |
318 | - # 出图的网格大小 | |
319 | - out_grid_x = (extent[2] - extent[0]) / (256 * 1.0) | |
320 | - out_grid_y = (extent[3] - extent[1]) / (256 * 1.0) | |
321 | - | |
322 | - out_off_x = int(math.ceil((inter_extent[0] - extent[0]) / out_grid_x)) | |
323 | - out_off_y = int(math.ceil((extent[3] - inter_extent[3]) / out_grid_y)) | |
324 | - | |
325 | - out_x_g = int(math.floor((inter_extent[2] - inter_extent[0]) / out_grid_x)) | |
326 | - out_y_g = int(math.floor((inter_extent[3] - inter_extent[1]) / out_grid_y)) | |
327 | - | |
328 | - # 相交部分在出图的哪个位置 | |
329 | - | |
330 | - overview_raster: ndarray = overview.ReadAsArray(off_x, off_y, x_g, y_g, out_x_g, | |
331 | - out_y_g) | |
332 | - | |
333 | - dat = numpy.zeros((256, 256), dtype=int) + 65536 | |
334 | - dat[out_off_y:out_off_y + out_y_g, out_off_x:out_off_x + out_x_g] = overview_raster | |
335 | - | |
336 | - empty = dat | |
337 | - | |
338 | - pixel_array[:, :, ceng] = empty | |
339 | - ceng += 1 | |
340 | - return pixel_array | |
341 | - | |
342 | - def determin_intersect(self,extent1,extent2): | |
343 | - g1 = GeometryAdapter.envelop_2_polygon(extent1) | |
344 | - g2 = GeometryAdapter.envelop_2_polygon(extent2) | |
345 | - return g1.Intersect(g2) | |
346 | - | |
347 | - api_doc = { | |
348 | - "tags": ["影像接口"], | |
349 | - "parameters": [ | |
350 | - {"name": "guid", | |
351 | - "in": "formData", | |
352 | - "type": "string"}, | |
353 | - {"name": "tilematrix", | |
354 | - "in": "formData", | |
355 | - "type": "string"}, | |
356 | - {"name": "tilerow", | |
357 | - "in": "formData", | |
358 | - "type": "string"}, | |
359 | - {"name": "tilecol", | |
360 | - "in": "formData", | |
361 | - "type": "string"}, | |
362 | - {"name": "format", | |
363 | - "in": "formData", | |
364 | - "type": "string"}, | |
365 | - {"name": "quality", | |
366 | - "in": "formData", | |
367 | - "type": "string"} | |
368 | - | |
369 | - ], | |
370 | - "responses": { | |
371 | - 200: { | |
372 | - "schema": { | |
373 | - "properties": { | |
374 | - } | |
375 | - } | |
376 | - } | |
377 | - } | |
378 | - } | |
379 | - | |
380 | -class MyThread(Thread): | |
381 | - def __init__(self,func,args=()): | |
382 | - super(MyThread,self).__init__() | |
383 | - self.func = func | |
384 | - self.args = args | |
385 | - def run(self): | |
386 | - self.result = self.func(*self.args) | |
387 | - def get_result(self): | |
388 | - try: | |
389 | - return self.result | |
390 | - except Exception: | |
391 | - return None | |
392 | - | |
393 | - | |
394 | - |
app/modules/service/__init__.py
0 → 100644
app/modules/service/address/__init__.py
0 → 100644
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
app/modules/service/image/data/t1/test.TIF
0 → 100644
不能预览此文件类型
... | ... | @@ -5,14 +5,11 @@ |
5 | 5 | |
6 | 6 | |
7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | -from thrift.transport import TSocket | |
9 | -from thrift.transport import TTransport | |
10 | -from thrift.protocol import TBinaryProtocol | |
8 | + | |
11 | 9 | import json |
12 | -from app.modules.image.ImageDataService import ImageDataService | |
13 | 10 | from app.util.component.FileProcess import FileProcess |
14 | 11 | import datetime |
15 | -from app.util.component.ThriftConnect import ThriftConnect | |
12 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
16 | 13 | import os |
17 | 14 | class Api(ApiTemplate): |
18 | 15 | ... | ... |
app/modules/service/image/file_tmp/f11036cb-121b-11ec-8b93-f82819efd8e6/Conf(CGCS2000).xml
0 → 100644
1 | +<?xml version="1.0" encoding="utf-8" ?><CacheInfo xsi:type='typens:CacheInfo' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.1'><TileCacheInfo xsi:type='typens:TileCacheInfo'><SpatialReference xsi:type='typens:GeographicCoordinateSystem'><WKT>GEOGCS["GCS_China_Geodetic_Coordinate_System_2000",DATUM["D_China_2000",SPHEROID["CGCS2000",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433],METADATA["China",73.66,7.16,134.85,53.59,0.0,0.0174532925199433,0.0,1067],AUTHORITY["EPSG",4490]]</WKT><XOrigin>-400</XOrigin><YOrigin>-400</YOrigin><XYScale>11258999068426.24</XYScale><ZOrigin>-100000</ZOrigin><ZScale>10000</ZScale><MOrigin>-100000</MOrigin><MScale>10000</MScale><XYTolerance>8.9831528411952133e-009</XYTolerance><ZTolerance>0.001</ZTolerance><MTolerance>0.001</MTolerance><HighPrecision>true</HighPrecision><LeftLongitude>-180</LeftLongitude><WKID>4490</WKID><LatestWKID>4490</LatestWKID></SpatialReference><TileOrigin xsi:type='typens:PointN'><X>-400</X><Y>399.99999999999977</Y></TileOrigin><TileCols>256</TileCols><TileRows>256</TileRows><DPI>96</DPI><PreciseDPI>96</PreciseDPI><LODInfos xsi:type='typens:ArrayOfLODInfo'><LODInfo xsi:type='typens:LODInfo'><LevelID>0</LevelID><Scale>1154287.473</Scale><Resolution>0.0027465820315218724</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>1</LevelID><Scale>577143.73640000005</Scale><Resolution>0.0013732910155229902</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>2</LevelID><Scale>288571.86820000003</Scale><Resolution>0.00068664550776149512</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>3</LevelID><Scale>144285.93410000001</Scale><Resolution>0.00034332275388074756</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>4</LevelID><Scale>72142.967059999995</Scale><Resolution>0.00017166137696416836</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>5</LevelID><Scale>36071.483529999998</Scale><Resolution>8.5830688482084179e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>6</LevelID><Scale>18035.741760000001</Scale><Resolution>4.2915344229144793e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>7</LevelID><Scale>9017.8708819999993</Scale><Resolution>2.1457672119331316e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>8</LevelID><Scale>4508.9354409999996</Scale><Resolution>1.0728836059665658e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>9</LevelID><Scale>2254.4677200000001</Scale><Resolution>5.3644180286430992e-006</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>10</LevelID><Scale>1127.23386</Scale><Resolution>2.6822090143215496e-006</Resolution></LODInfo></LODInfos></TileCacheInfo><TileImageInfo xsi:type='typens:TileImageInfo'><CacheTileFormat>PNG32</CacheTileFormat><CompressionQuality>0</CompressionQuality><Antialiasing>false</Antialiasing></TileImageInfo><CacheStorageInfo xsi:type='typens:CacheStorageInfo'><StorageFormat>esriMapCacheStorageModeCompact</StorageFormat><PacketSize>128</PacketSize></CacheStorageInfo></CacheInfo> | |
\ No newline at end of file | ... | ... |
... | ... | @@ -7,7 +7,7 @@ |
7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | 8 | from app.util.component.ModelVisitor import ModelVisitor |
9 | 9 | |
10 | -from app.modules.image.models import Image | |
10 | +from app.modules.service.image.models import Image | |
11 | 11 | from sqlalchemy import or_,and_ |
12 | 12 | class Api(ApiTemplate): |
13 | 13 | ... | ... |
... | ... | @@ -7,8 +7,7 @@ |
7 | 7 | from osgeo import gdal,ogr,osr |
8 | 8 | from osgeo.gdal import Dataset,Band |
9 | 9 | from app.util.component.ApiTemplate import ApiTemplate |
10 | -from app.util.component.ThriftConnect import ThriftConnect | |
11 | - | |
10 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
12 | 11 | import json |
13 | 12 | from .models import Image |
14 | 13 | import datetime | ... | ... |
... | ... | @@ -6,7 +6,7 @@ |
6 | 6 | |
7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | 8 | from app.util.component.ModelVisitor import ModelVisitor |
9 | -from app.modules.image.models import ImageService | |
9 | +from app.modules.service.image.models import ImageService | |
10 | 10 | from sqlalchemy import or_ |
11 | 11 | |
12 | 12 | class Api(ApiTemplate): | ... | ... |
... | ... | @@ -3,7 +3,7 @@ |
3 | 3 | #createtime: 2021/7/19 |
4 | 4 | #email: nheweijun@sina.com |
5 | 5 | |
6 | -from app.modules.image.models import ImageService,Image | |
6 | +from app.modules.service.image.models import ImageService,Image | |
7 | 7 | from app.models import db |
8 | 8 | from app.util.component.ApiTemplate import ApiTemplate |
9 | 9 | import uuid | ... | ... |
... | ... | @@ -16,7 +16,7 @@ from PIL import Image |
16 | 16 | |
17 | 17 | import time |
18 | 18 | import cv2 |
19 | -from app.modules.image.models import ImageService,Image | |
19 | +from app.modules.service.image.models import ImageService,Image | |
20 | 20 | from app.models import db |
21 | 21 | from app.util.component.ApiTemplate import ApiTemplate |
22 | 22 | import uuid |
... | ... | @@ -30,7 +30,7 @@ from kazoo.client import KazooClient |
30 | 30 | from app import GLOBAL_DIC |
31 | 31 | from threading import Thread |
32 | 32 | |
33 | -from app.util.component.ThriftConnect import ThriftConnect | |
33 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
34 | 34 | from flask import current_app |
35 | 35 | import gzip |
36 | 36 | import random | ... | ... |
... | ... | @@ -13,10 +13,10 @@ from flask import Response |
13 | 13 | import random |
14 | 14 | import time |
15 | 15 | import cv2 |
16 | -from app.modules.image.models import ImageService | |
16 | +from app.modules.service.image.models import ImageService | |
17 | 17 | from app.util.component.ApiTemplate import ApiTemplate |
18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
19 | -from app.util.component.ThriftConnect import ThriftConnect | |
19 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
20 | 20 | from app.util.component.ParameterUtil import ParameterUtil |
21 | 21 | import json |
22 | 22 | from kazoo.client import KazooClient | ... | ... |
... | ... | @@ -6,7 +6,7 @@ |
6 | 6 | from thrift.transport import TSocket |
7 | 7 | from thrift.transport import TTransport |
8 | 8 | from thrift.protocol import TBinaryProtocol |
9 | -from app.modules.image.ImageDataService import ImageDataService | |
9 | +from app.modules.service.image.ImageDataService import ImageDataService | |
10 | 10 | |
11 | 11 | class ThriftConnect: |
12 | 12 | ''' | ... | ... |
app/modules/service/image/util/__init__.py
0 → 100644
app/modules/service/wms/__init__.py
0 → 100644
app/modules/service/wmts/__init__.py
0 → 100644
请
注册
或
登录
后发表评论