正在显示
87 个修改的文件
包含
179 行增加
和
547 行删除
@@ -3,7 +3,6 @@ import decimal | @@ -3,7 +3,6 @@ import decimal | ||
3 | from flask import Flask as _Flask | 3 | from flask import Flask as _Flask |
4 | from flask.json import JSONEncoder as _JSONEncoder | 4 | from flask.json import JSONEncoder as _JSONEncoder |
5 | from flask_cors import CORS | 5 | from flask_cors import CORS |
6 | -import time | ||
7 | 6 | ||
8 | from sqlalchemy.sql.expression import true | 7 | from sqlalchemy.sql.expression import true |
9 | import configure | 8 | import configure |
@@ -12,24 +11,14 @@ from app.util import find_class | @@ -12,24 +11,14 @@ from app.util import find_class | ||
12 | from app.models import db, Table, InsertingLayerName, Database, DES, Task | 11 | from app.models import db, Table, InsertingLayerName, Database, DES, Task |
13 | from app.modules.auth.oauth2 import config_oauth, myCodeIDToken | 12 | from app.modules.auth.oauth2 import config_oauth, myCodeIDToken |
14 | from flasgger import Swagger | 13 | from flasgger import Swagger |
15 | -# from rtree import index | ||
16 | import logging | 14 | import logging |
17 | -from sqlalchemy.orm import Session | ||
18 | -import multiprocessing | ||
19 | from app.util.component.EntryData import EntryData | 15 | from app.util.component.EntryData import EntryData |
20 | from app.util.component.EntryDataVacuate import EntryDataVacuate | 16 | from app.util.component.EntryDataVacuate import EntryDataVacuate |
21 | -import json | ||
22 | import threading | 17 | import threading |
23 | -import traceback | ||
24 | -from sqlalchemy import distinct | ||
25 | -import uuid | ||
26 | -from osgeo.ogr import DataSource | ||
27 | -import datetime | ||
28 | -from sqlalchemy import or_ | ||
29 | from app.util.component.StructuredPrint import StructurePrint | 18 | from app.util.component.StructuredPrint import StructurePrint |
30 | from app.util.component.PGUtil import PGUtil | 19 | from app.util.component.PGUtil import PGUtil |
31 | import os | 20 | import os |
32 | - | 21 | +from app.modules.data.io.data_entry_center import data_entry_center |
33 | 22 | ||
34 | class JSONEncoder(_JSONEncoder): | 23 | class JSONEncoder(_JSONEncoder): |
35 | """ | 24 | """ |
@@ -105,7 +94,6 @@ def create_app(): | @@ -105,7 +94,6 @@ def create_app(): | ||
105 | app.register_blueprint(api.bp) | 94 | app.register_blueprint(api.bp) |
106 | 95 | ||
107 | # 入库监测线程 | 96 | # 入库监测线程 |
108 | - | ||
109 | @app.before_first_request | 97 | @app.before_first_request |
110 | def data_entry_process(): | 98 | def data_entry_process(): |
111 | StructurePrint.print("start listen") | 99 | StructurePrint.print("start listen") |
@@ -120,99 +108,3 @@ def create_app(): | @@ -120,99 +108,3 @@ def create_app(): | ||
120 | 108 | ||
121 | return app | 109 | return app |
122 | 110 | ||
123 | - | ||
124 | -def data_entry_center(): | ||
125 | - running_dict = {} | ||
126 | - sys_session: Session = PGUtil.get_db_session( | ||
127 | - configure.SQLALCHEMY_DATABASE_URI) | ||
128 | - | ||
129 | - while True: | ||
130 | - | ||
131 | - try: | ||
132 | - time.sleep(3) | ||
133 | - | ||
134 | - # 已经结束的进程 从监测中删除 | ||
135 | - remove_process = [] | ||
136 | - | ||
137 | - # structured_print(running_dict.__len__().__str__()) | ||
138 | - | ||
139 | - for process, layer_names in running_dict.items(): | ||
140 | - if not process.is_alive(): | ||
141 | - for l in layer_names: | ||
142 | - inserted = sys_session.query( | ||
143 | - InsertingLayerName).filter_by(name=l).one_or_none() | ||
144 | - if inserted: | ||
145 | - sys_session.delete(inserted) | ||
146 | - sys_session.commit() | ||
147 | - remove_process.append(process) | ||
148 | - for process in remove_process: | ||
149 | - running_dict.pop(process) | ||
150 | - | ||
151 | - # StructurePrint.print("listening...") | ||
152 | - | ||
153 | - # 入库进程少于阈值,开启入库进程 | ||
154 | - | ||
155 | - inter_size = sys_session.query( | ||
156 | - distinct(InsertingLayerName.task_guid)).count() | ||
157 | - | ||
158 | - if inter_size < configure.entry_data_thread: | ||
159 | - # 锁表啊 | ||
160 | - ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | ||
161 | - Task.create_time).with_lockmode("update").limit(1).one_or_none() | ||
162 | - if ready_task: | ||
163 | - | ||
164 | - try: | ||
165 | - parameter = json.loads(ready_task.parameter) | ||
166 | - StructurePrint.print("检测到入库任务") | ||
167 | - ready_task.state = 2 | ||
168 | - ready_task.process = "入库中" | ||
169 | - sys_session.commit() | ||
170 | - | ||
171 | - metas: list = json.loads( | ||
172 | - parameter.get("meta").__str__()) | ||
173 | - parameter["meta"] = metas | ||
174 | - | ||
175 | - database = sys_session.query(Database).filter_by( | ||
176 | - guid=ready_task.database_guid).one_or_none() | ||
177 | - pg_ds: DataSource = PGUtil.open_pg_data_source( | ||
178 | - 1, DES.decode(database.sqlalchemy_uri)) | ||
179 | - | ||
180 | - this_task_layer = [] | ||
181 | - for meta in metas: | ||
182 | - overwrite = parameter.get("overwrite", "no") | ||
183 | - | ||
184 | - for layer_name_origin, layer_name in meta.get("layer").items(): | ||
185 | - origin_name = layer_name | ||
186 | - no = 1 | ||
187 | - | ||
188 | - while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | ||
189 | - layer_name = origin_name + "_{}".format(no) | ||
190 | - no += 1 | ||
191 | - | ||
192 | - # 添加到正在入库的列表中 | ||
193 | - iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | ||
194 | - task_guid=ready_task.guid, | ||
195 | - name=layer_name) | ||
196 | - | ||
197 | - sys_session.add(iln) | ||
198 | - sys_session.commit() | ||
199 | - this_task_layer.append(layer_name) | ||
200 | - # 修改表名 | ||
201 | - meta["layer"][layer_name_origin] = layer_name | ||
202 | - | ||
203 | - pg_ds.Destroy() | ||
204 | - entry_data_process = multiprocessing.Process( | ||
205 | - target=EntryDataVacuate().entry, args=(parameter,)) | ||
206 | - entry_data_process.start() | ||
207 | - running_dict[entry_data_process] = this_task_layer | ||
208 | - except Exception as e: | ||
209 | - sys_session.query(Task).filter_by(guid=ready_task.guid).update( | ||
210 | - {"state": -1, "process": "入库失败"}) | ||
211 | - sys_session.commit() | ||
212 | - StructurePrint.print(e.__str__(), "error") | ||
213 | - else: | ||
214 | - # 解表啊 | ||
215 | - sys_session.commit() | ||
216 | - except Exception as e: | ||
217 | - sys_session.commit() | ||
218 | - StructurePrint.print(e.__str__(), "error") |
@@ -34,13 +34,14 @@ class DES(): | @@ -34,13 +34,14 @@ class DES(): | ||
34 | db = SQLAlchemy() | 34 | db = SQLAlchemy() |
35 | 35 | ||
36 | # 动态加载Model | 36 | # 动态加载Model |
37 | -module_dir = "modules" | ||
38 | -model_pkg = "models" | ||
39 | current_dir = os.path.abspath(os.path.dirname(__file__)) | 37 | current_dir = os.path.abspath(os.path.dirname(__file__)) |
40 | -for pkg in glob.glob('%s/%s/*/%s' % (current_dir,module_dir,model_pkg)) : | ||
41 | - module = os.path.basename(os.path.dirname(pkg)) | ||
42 | - base_name = os.path.basename(pkg).split('.')[0] | ||
43 | - pkg_name = ".".join(["app.modules",module,base_name]) | 38 | +pkgs = list(glob.glob('%s/modules/*/models' % (current_dir))) |
39 | +pkgs.extend(list(glob.glob('%s/modules/*/*/models' % (current_dir)))) | ||
40 | + | ||
41 | +for pkg in pkgs : | ||
42 | + pkg = os.path.normpath(pkg) | ||
43 | + node_list = pkg.split(os.path.sep) | ||
44 | + pkg_name = ".".join(node_list[node_list.index("app"):]) | ||
44 | try: | 45 | try: |
45 | __import__(pkg_name) | 46 | __import__(pkg_name) |
46 | except: | 47 | except: |
@@ -199,21 +200,21 @@ class InsertingLayerName(db.Model): | @@ -199,21 +200,21 @@ class InsertingLayerName(db.Model): | ||
199 | name = Column(String(256)) | 200 | name = Column(String(256)) |
200 | 201 | ||
201 | 202 | ||
202 | -# class Service(db.Model): | ||
203 | -# ''' | ||
204 | -# ''' | ||
205 | -# __tablename__ = 'dmdms_service' | ||
206 | -# guid = Column(String(256), primary_key=True) | ||
207 | -# name=Column(String(256)) | ||
208 | -# alias = Column(String(256)) | ||
209 | -# #服务状态 | ||
210 | -# state= Column(Integer) | ||
211 | -# create_time = Column(DateTime) | ||
212 | -# update_time = Column(DateTime) | ||
213 | -# #服务描述 | ||
214 | -# description = Column(Text) | ||
215 | -# #服务节点 | ||
216 | -# node = Column(Integer) | ||
217 | -# #服务缩略图 | ||
218 | -# overview = Column(Binary) | 203 | +class Service(db.Model): |
204 | + ''' | ||
205 | + ''' | ||
206 | + __tablename__ = 'dmdms_service' | ||
207 | + guid = Column(String(256), primary_key=True) | ||
208 | + name=Column(String(256)) | ||
209 | + alias = Column(String(256)) | ||
210 | + #服务状态 | ||
211 | + state= Column(Integer) | ||
212 | + create_time = Column(DateTime) | ||
213 | + update_time = Column(DateTime) | ||
214 | + #服务描述 | ||
215 | + description = Column(Text) | ||
216 | + #服务节点 | ||
217 | + node = Column(Integer) | ||
218 | + #服务缩略图 | ||
219 | + overview = Column(Binary) | ||
219 | 220 |
app/modules/data/__init__.py
0 → 100644
@@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate | @@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate | ||
8 | import uuid | 8 | import uuid |
9 | from . import database_test | 9 | from . import database_test |
10 | # from app.util import open_pg_data_source | 10 | # from app.util import open_pg_data_source |
11 | -from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn,Feature,Geometry | 11 | +from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn |
12 | from sqlalchemy.orm import Session | 12 | from sqlalchemy.orm import Session |
13 | from sqlalchemy import create_engine | 13 | from sqlalchemy import create_engine |
14 | from sqlalchemy.orm import sessionmaker | 14 | from sqlalchemy.orm import sessionmaker |
@@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil | @@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil | ||
17 | from app.util.component.SQLUtil import SQLUtil | 17 | from app.util.component.SQLUtil import SQLUtil |
18 | from app.util.component.GeometryAdapter import GeometryAdapter | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
19 | from app.util.component.StructuredPrint import StructurePrint | 19 | from app.util.component.StructuredPrint import StructurePrint |
20 | -import configure | 20 | + |
21 | + | ||
21 | class Api(ApiTemplate): | 22 | class Api(ApiTemplate): |
22 | api_name = "注册数据库" | 23 | api_name = "注册数据库" |
23 | def process(self): | 24 | def process(self): |
@@ -6,15 +6,13 @@ from flasgger import swag_from | @@ -6,15 +6,13 @@ from flasgger import swag_from | ||
6 | from flask import Blueprint | 6 | from flask import Blueprint |
7 | from app.util import BlueprintApi | 7 | from app.util import BlueprintApi |
8 | 8 | ||
9 | -from flask import send_from_directory,current_app,send_file | 9 | +from flask import send_from_directory |
10 | import os | 10 | import os |
11 | -import shutil | ||
12 | from . import data_download | 11 | from . import data_download |
13 | from . import get_meta | 12 | from . import get_meta |
14 | from . import data_entry_by_meta | 13 | from . import data_entry_by_meta |
15 | from . import get_data_list | 14 | from . import get_data_list |
16 | -from flask import after_this_request | ||
17 | -import time | 15 | + |
18 | 16 | ||
19 | class DataManager(BlueprintApi): | 17 | class DataManager(BlueprintApi): |
20 | 18 |
app/modules/data/io/data_entry_center.py
0 → 100644
1 | +# coding=utf-8 | ||
2 | +#author: 4N | ||
3 | +#createtime: 2021/9/14 | ||
4 | +#email: nheweijun@sina.com | ||
5 | + | ||
6 | +import configure | ||
7 | +from app.models import InsertingLayerName, Database, DES, Task | ||
8 | +from sqlalchemy.orm import Session | ||
9 | +import multiprocessing | ||
10 | +from app.util.component.EntryDataVacuate import EntryDataVacuate | ||
11 | +import json | ||
12 | +from sqlalchemy import distinct | ||
13 | +import uuid | ||
14 | +from osgeo.ogr import DataSource | ||
15 | +from app.util.component.StructuredPrint import StructurePrint | ||
16 | +from app.util.component.PGUtil import PGUtil | ||
17 | +import time | ||
18 | + | ||
19 | +def data_entry_center(): | ||
20 | + running_dict = {} | ||
21 | + sys_session: Session = PGUtil.get_db_session( | ||
22 | + configure.SQLALCHEMY_DATABASE_URI) | ||
23 | + | ||
24 | + while True: | ||
25 | + | ||
26 | + try: | ||
27 | + time.sleep(3) | ||
28 | + | ||
29 | + # 已经结束的进程 从监测中删除 | ||
30 | + remove_process = [] | ||
31 | + | ||
32 | + # structured_print(running_dict.__len__().__str__()) | ||
33 | + | ||
34 | + for process, layer_names in running_dict.items(): | ||
35 | + if not process.is_alive(): | ||
36 | + for l in layer_names: | ||
37 | + inserted = sys_session.query( | ||
38 | + InsertingLayerName).filter_by(name=l).one_or_none() | ||
39 | + if inserted: | ||
40 | + sys_session.delete(inserted) | ||
41 | + sys_session.commit() | ||
42 | + remove_process.append(process) | ||
43 | + for process in remove_process: | ||
44 | + running_dict.pop(process) | ||
45 | + | ||
46 | + # StructurePrint.print("listening...") | ||
47 | + | ||
48 | + # 入库进程少于阈值,开启入库进程 | ||
49 | + | ||
50 | + inter_size = sys_session.query( | ||
51 | + distinct(InsertingLayerName.task_guid)).count() | ||
52 | + | ||
53 | + if inter_size < configure.entry_data_thread: | ||
54 | + # 锁表啊 | ||
55 | + ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | ||
56 | + Task.create_time).with_lockmode("update").limit(1).one_or_none() | ||
57 | + if ready_task: | ||
58 | + | ||
59 | + try: | ||
60 | + parameter = json.loads(ready_task.parameter) | ||
61 | + StructurePrint.print("检测到入库任务") | ||
62 | + ready_task.state = 2 | ||
63 | + ready_task.process = "入库中" | ||
64 | + sys_session.commit() | ||
65 | + | ||
66 | + metas: list = json.loads( | ||
67 | + parameter.get("meta").__str__()) | ||
68 | + parameter["meta"] = metas | ||
69 | + | ||
70 | + database = sys_session.query(Database).filter_by( | ||
71 | + guid=ready_task.database_guid).one_or_none() | ||
72 | + pg_ds: DataSource = PGUtil.open_pg_data_source( | ||
73 | + 1, DES.decode(database.sqlalchemy_uri)) | ||
74 | + | ||
75 | + this_task_layer = [] | ||
76 | + for meta in metas: | ||
77 | + overwrite = parameter.get("overwrite", "no") | ||
78 | + | ||
79 | + for layer_name_origin, layer_name in meta.get("layer").items(): | ||
80 | + origin_name = layer_name | ||
81 | + no = 1 | ||
82 | + | ||
83 | + while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | ||
84 | + layer_name = origin_name + "_{}".format(no) | ||
85 | + no += 1 | ||
86 | + | ||
87 | + # 添加到正在入库的列表中 | ||
88 | + iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | ||
89 | + task_guid=ready_task.guid, | ||
90 | + name=layer_name) | ||
91 | + | ||
92 | + sys_session.add(iln) | ||
93 | + sys_session.commit() | ||
94 | + this_task_layer.append(layer_name) | ||
95 | + # 修改表名 | ||
96 | + meta["layer"][layer_name_origin] = layer_name | ||
97 | + | ||
98 | + pg_ds.Destroy() | ||
99 | + entry_data_process = multiprocessing.Process( | ||
100 | + target=EntryDataVacuate().entry, args=(parameter,)) | ||
101 | + entry_data_process.start() | ||
102 | + running_dict[entry_data_process] = this_task_layer | ||
103 | + except Exception as e: | ||
104 | + sys_session.query(Task).filter_by(guid=ready_task.guid).update( | ||
105 | + {"state": -1, "process": "入库失败"}) | ||
106 | + sys_session.commit() | ||
107 | + StructurePrint.print(e.__str__(), "error") | ||
108 | + else: | ||
109 | + # 解表啊 | ||
110 | + sys_session.commit() | ||
111 | + except Exception as e: | ||
112 | + sys_session.commit() | ||
113 | + StructurePrint.print(e.__str__(), "error") |
app/modules/image/image_tilebac.py
deleted
100644 → 0
1 | -# coding=utf-8 | ||
2 | -#author: 4N | ||
3 | -#createtime: 2021/3/24 | ||
4 | -#email: nheweijun@sina.com | ||
5 | - | ||
6 | -from app.util import * | ||
7 | -import traceback | ||
8 | -from osgeo import gdal | ||
9 | -from osgeo.gdal import * | ||
10 | -from numpy import ndarray | ||
11 | -import numpy | ||
12 | -from flask import Response | ||
13 | -import io | ||
14 | -import os | ||
15 | -from PIL import Image | ||
16 | - | ||
17 | -import time | ||
18 | -import cv2 | ||
19 | -from app.modules.image.models import ImageService,Image | ||
20 | -from app.models import db | ||
21 | -from app.util.component.ApiTemplate import ApiTemplate | ||
22 | -import uuid | ||
23 | -from app.util.component.SliceScheme import SliceScheme | ||
24 | -from app.util.component.FileProcess import FileProcess | ||
25 | -from app.util.component.ParameterUtil import ParameterUtil | ||
26 | -from app.util.component.GeometryAdapter import GeometryAdapter | ||
27 | -import os | ||
28 | -import json | ||
29 | -from kazoo.client import KazooClient | ||
30 | -from app import GLOBAL_DIC | ||
31 | -from threading import Thread | ||
32 | -from thrift.transport import TSocket | ||
33 | -from thrift.transport import TTransport | ||
34 | -from thrift.protocol import TBinaryProtocol | ||
35 | -from .ImageDataService import ImageDataService | ||
36 | -from flask import current_app | ||
37 | -import gzip | ||
38 | -import random | ||
39 | -class Api(ApiTemplate): | ||
40 | - | ||
41 | - api_name = "切片" | ||
42 | - | ||
43 | - def __init__(self,guid,level, row, col): | ||
44 | - super().__init__() | ||
45 | - self.guid = guid | ||
46 | - self.level = level | ||
47 | - self.row = row | ||
48 | - self.col = col | ||
49 | - | ||
50 | - def process(self): | ||
51 | - | ||
52 | - result = {} | ||
53 | - parameter: dict = self.para | ||
54 | - | ||
55 | - try: | ||
56 | - if parameter.get("guid"): | ||
57 | - self.guid = parameter.get("guid") | ||
58 | - | ||
59 | - image_service = ImageService.query.filter_by(guid = self.guid).one_or_none() | ||
60 | - images = image_service.images.all() | ||
61 | - | ||
62 | - zoo = GLOBAL_DIC.get("zookeeper") | ||
63 | - if zoo is None: | ||
64 | - zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100) | ||
65 | - zoo.start() | ||
66 | - GLOBAL_DIC["zookeeper"] = zoo | ||
67 | - else : | ||
68 | - if not zoo.connected: | ||
69 | - zoo.start() | ||
70 | - | ||
71 | - bands = [1, 2, 3] | ||
72 | - | ||
73 | - | ||
74 | - # 转换参数 | ||
75 | - parameter = ParameterUtil.to_lower(parameter) | ||
76 | - | ||
77 | - | ||
78 | - if parameter.get("tilematrix"): | ||
79 | - if parameter.get("tilematrix").__contains__(":"): | ||
80 | - self.level = int(parameter.get("tilematrix").split(":")[-1]) | ||
81 | - else: | ||
82 | - self.level = int(parameter.get("tilematrix")) | ||
83 | - if parameter.get("tilerow"): | ||
84 | - self.row = int(parameter.get("tilerow")) | ||
85 | - if parameter.get("tilecol"): | ||
86 | - self.col = int(parameter.get("tilecol")) | ||
87 | - | ||
88 | - image_type = parameter.get("format") if parameter.get("format") else "image/png" | ||
89 | - quality = int(parameter.get("quality")) if parameter.get("quality") else 30 | ||
90 | - slice_para = json.loads(image_service.slice_scheme) | ||
91 | - extent = SliceScheme.get_polygon(slice_para, self.level, self.row, self.col) | ||
92 | - | ||
93 | - | ||
94 | - | ||
95 | - # 多线程获取分布式数据 | ||
96 | - | ||
97 | - intersect_image = [im for im in images if self.determin_intersect(json.loads(im.extent),extent)] | ||
98 | - | ||
99 | - pixel_array = numpy.zeros((256, 256,3), dtype=int) | ||
100 | - | ||
101 | - if len(intersect_image) > 1: | ||
102 | - | ||
103 | - # 结果矩阵 | ||
104 | - empty_list = [numpy.zeros((256,256), dtype=int) + 65536, | ||
105 | - numpy.zeros((256,256), dtype=int) + 65536, | ||
106 | - numpy.zeros((256,256), dtype=int) + 65536] | ||
107 | - | ||
108 | - pixel_array = numpy.zeros((256,256,3), dtype=int) | ||
109 | - thread_list = [] | ||
110 | - | ||
111 | - for image in intersect_image: | ||
112 | - #该影像的服务器,随机选取一个 | ||
113 | - image_servers = image.server.split(",") | ||
114 | - indx = int(random.random()*len(image_servers)) | ||
115 | - image_server = image_servers[indx] | ||
116 | - | ||
117 | - thread: MyThread = MyThread(self.get_data, args=(zoo,image_server,image,extent,bands)) | ||
118 | - thread.start() | ||
119 | - thread_list.append(thread) | ||
120 | - | ||
121 | - | ||
122 | - for thread in thread_list: | ||
123 | - thread.join() | ||
124 | - data = thread.get_result() | ||
125 | - | ||
126 | - # 掩膜在中央接口生成,合图 | ||
127 | - mask = numpy.zeros((256,256), dtype=int) | ||
128 | - mask2 = numpy.zeros((256,256), dtype=int) | ||
129 | - jizhun = data[:, :, 0] | ||
130 | - mask[jizhun == 65536] = 1 | ||
131 | - mask[jizhun != 65536] = 0 | ||
132 | - mask2[jizhun == 65536] = 0 | ||
133 | - mask2[jizhun != 65536] = 1 | ||
134 | - # 掩膜计算 | ||
135 | - for i, d in enumerate(empty_list): | ||
136 | - empty_list[i] = empty_list[i] * mask + data[:, :, i] * mask2 | ||
137 | - for ii in [0, 1, 2]: | ||
138 | - # opencv 颜色排序为GBR | ||
139 | - pixel_array[:, :, 2 - ii] = empty_list[ii] | ||
140 | - | ||
141 | - elif len(intersect_image) == 1: | ||
142 | - | ||
143 | - # 该影像的服务器,随机选取一个 | ||
144 | - image = intersect_image[0] | ||
145 | - image_servers = image.server.split(",") | ||
146 | - indx = int(random.random() * len(image_servers)) | ||
147 | - image_server = image_servers[indx] | ||
148 | - pixel_array_t = self.get_data(zoo, image_server, image, extent, bands) | ||
149 | - pixel_array = numpy.zeros((256,256, 3), dtype=int) | ||
150 | - for ii in [0, 1, 2]: | ||
151 | - # opencv 颜色排序为GBR | ||
152 | - pixel_array[:, :, 2 - ii] = pixel_array_t[:, :, ii] | ||
153 | - | ||
154 | - else: | ||
155 | - # 结果矩阵 | ||
156 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) + 65536 | ||
157 | - | ||
158 | - #将图片生成在内存中,然后直接返回response | ||
159 | - im_data = self.create_by_opencv(image_type, pixel_array, quality) | ||
160 | - | ||
161 | - return Response(im_data, mimetype=image_type.lower()) | ||
162 | - | ||
163 | - except Exception as e: | ||
164 | - print(traceback.format_exc()) | ||
165 | - result["state"] = -1 | ||
166 | - result["message"] = e.__str__() | ||
167 | - return result | ||
168 | - | ||
169 | - | ||
170 | - def determine_level(self,xysize,origin_extent,extent,max_level): | ||
171 | - ''' | ||
172 | - 根据范围判断调用金字塔的哪一层 | ||
173 | - :param xysize: | ||
174 | - :param origin_extent: | ||
175 | - :param extent: | ||
176 | - :param max_level: | ||
177 | - :return: | ||
178 | - ''' | ||
179 | - x = xysize[0] | ||
180 | - y = xysize[1] | ||
181 | - level = -1 | ||
182 | - pixel = x*y * (((extent[2]-extent[0])*(extent[3]-extent[1]))/((origin_extent[2]-origin_extent[0])*(origin_extent[3]-origin_extent[1]))) | ||
183 | - while pixel>100000 and level<max_level-1: | ||
184 | - level+=1 | ||
185 | - x=x/2 | ||
186 | - y=y/2 | ||
187 | - pixel = x * y * (((extent[2] - extent[0]) * (extent[3] - extent[1])) / ( | ||
188 | - (origin_extent[2] - origin_extent[0]) * (origin_extent[3] - origin_extent[1]))) | ||
189 | - return level | ||
190 | - | ||
191 | - def create_by_opencv(self,image_type, pixel_array, quality): | ||
192 | - | ||
193 | - if image_type.__eq__("image/jpeg") or image_type.__eq__("image/jpg"): | ||
194 | - r, buf = cv2.imencode(".jpg", pixel_array, [cv2.IMWRITE_JPEG_QUALITY, quality]) | ||
195 | - image_out = buf.tobytes() | ||
196 | - else: | ||
197 | - four = numpy.zeros((256, 256), dtype=int) + 255 | ||
198 | - four[pixel_array[:, :, 0] == 65536] = 0 | ||
199 | - r, buf = cv2.imencode(".png", numpy.dstack((pixel_array, four))) | ||
200 | - image_out = buf.tobytes() | ||
201 | - return image_out | ||
202 | - | ||
203 | - def get_data(self,zoo,image_server,image,extent,bands): | ||
204 | - | ||
205 | - if image_server.__eq__("本地服务器"): | ||
206 | - data = self.get_local_data(image, extent, bands) | ||
207 | - else: | ||
208 | - ser = image_server | ||
209 | - if zoo.exists("/rpc/{}".format(ser)): | ||
210 | - data = self.get_remote_data(image, extent) | ||
211 | - else: | ||
212 | - data = numpy.zeros((256, 256, 3), dtype=int) + 65536 | ||
213 | - return data | ||
214 | - | ||
215 | - | ||
216 | - def get_remote_data(self,image,extent,bands): | ||
217 | - ''' | ||
218 | - 通过RPC获取远程数据 | ||
219 | - :param image: | ||
220 | - :param extent: | ||
221 | - :param bands: | ||
222 | - :return: | ||
223 | - ''' | ||
224 | - | ||
225 | - transport = TSocket.TSocket(image.host, image.port) | ||
226 | - transport = TTransport.TBufferedTransport(transport) | ||
227 | - protocol = TBinaryProtocol.TBinaryProtocol(transport) | ||
228 | - client = ImageDataService.Client(protocol) | ||
229 | - transport.open() | ||
230 | - t1 = time.time() | ||
231 | - data = client.getData(image.path, extent, json.loads(image.extent), bands,256,256) | ||
232 | - transport.close() | ||
233 | - current_app.logger.info("time {}".format(time.time() - t1)) | ||
234 | - | ||
235 | - data = gzip.decompress(data) | ||
236 | - data = numpy.frombuffer(data, dtype=int) | ||
237 | - data = data.reshape((256, 256, 3)) | ||
238 | - | ||
239 | - return data | ||
240 | - | ||
241 | - def get_local_data(self,image,extent,bands): | ||
242 | - ''' | ||
243 | - 获取本地数据 | ||
244 | - :param image: | ||
245 | - :param extent: | ||
246 | - :param bands: | ||
247 | - :return: | ||
248 | - ''' | ||
249 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) | ||
250 | - ceng = 0 | ||
251 | - img: Dataset = gdal.Open(image.path, 0) | ||
252 | - t1 = time.time() | ||
253 | - for band in bands: | ||
254 | - | ||
255 | - # 自决定金字塔等级 | ||
256 | - xysize = [img.RasterXSize, img.RasterYSize] | ||
257 | - origin_extent = image.extent | ||
258 | - band_data: Band = img.GetRasterBand(band) | ||
259 | - max_level = band_data.GetOverviewCount() | ||
260 | - | ||
261 | - # 超出空间范围 | ||
262 | - if extent[2] < origin_extent[0] or extent[0] > origin_extent[2] or extent[1] > origin_extent[ | ||
263 | - 3] or extent[3] < origin_extent[1]: | ||
264 | - empty = numpy.zeros((256, 256), dtype=int) + 65536 | ||
265 | - # 空间范围相交 | ||
266 | - else: | ||
267 | - image_level = self.determine_level(xysize, origin_extent, extent, max_level) | ||
268 | - | ||
269 | - if image_level == -1: | ||
270 | - overview = band_data | ||
271 | - else: | ||
272 | - try: | ||
273 | - overview: Band = band_data.GetOverview(image_level) | ||
274 | - except: | ||
275 | - raise Exception("该影像不存在该级别的金字塔数据!") | ||
276 | - ox = overview.XSize | ||
277 | - oy = overview.YSize | ||
278 | - | ||
279 | - # 网格大小 | ||
280 | - grid_x = (origin_extent[2] - origin_extent[0]) / (ox * 1.0) | ||
281 | - grid_y = (origin_extent[3] - origin_extent[1]) / (oy * 1.0) | ||
282 | - | ||
283 | - # 完全在影像范围内 | ||
284 | - if extent[0] > origin_extent[0] and extent[1] > origin_extent[1] and extent[2] < \ | ||
285 | - origin_extent[2] and extent[3] < origin_extent[3]: | ||
286 | - | ||
287 | - # 网格偏移量 | ||
288 | - off_x = math.floor((extent[0] - origin_extent[0]) / grid_x) | ||
289 | - off_y = math.floor((origin_extent[3] - extent[3]) / grid_y) | ||
290 | - | ||
291 | - # 截取后网格个数 | ||
292 | - x_g = math.ceil((extent[2] - extent[0]) / grid_x) | ||
293 | - | ||
294 | - y_g = math.ceil((extent[3] - extent[1]) / grid_y) | ||
295 | - | ||
296 | - empty = overview.ReadAsArray(off_x, off_y, x_g, y_g, 256, 256) | ||
297 | - | ||
298 | - | ||
299 | - # 部分相交 | ||
300 | - else: | ||
301 | - | ||
302 | - inter_extent = [0, 0, 0, 0] | ||
303 | - inter_extent[0] = origin_extent[0] if origin_extent[0] > extent[0] else extent[0] | ||
304 | - inter_extent[1] = origin_extent[1] if origin_extent[1] > extent[1] else extent[1] | ||
305 | - inter_extent[2] = origin_extent[2] if origin_extent[2] < extent[2] else extent[2] | ||
306 | - inter_extent[3] = origin_extent[3] if origin_extent[3] < extent[3] else extent[3] | ||
307 | - | ||
308 | - # 网格偏移量 | ||
309 | - off_x = math.floor((inter_extent[0] - origin_extent[0]) / grid_x) | ||
310 | - off_y = math.floor((origin_extent[3] - inter_extent[3]) / grid_y) | ||
311 | - | ||
312 | - # 截取后网格个数 | ||
313 | - x_g = math.floor((inter_extent[2] - inter_extent[0]) / grid_x) | ||
314 | - y_g = math.floor((inter_extent[3] - inter_extent[1]) / grid_y) | ||
315 | - | ||
316 | - # 相对于出图的偏移量 | ||
317 | - | ||
318 | - # 出图的网格大小 | ||
319 | - out_grid_x = (extent[2] - extent[0]) / (256 * 1.0) | ||
320 | - out_grid_y = (extent[3] - extent[1]) / (256 * 1.0) | ||
321 | - | ||
322 | - out_off_x = int(math.ceil((inter_extent[0] - extent[0]) / out_grid_x)) | ||
323 | - out_off_y = int(math.ceil((extent[3] - inter_extent[3]) / out_grid_y)) | ||
324 | - | ||
325 | - out_x_g = int(math.floor((inter_extent[2] - inter_extent[0]) / out_grid_x)) | ||
326 | - out_y_g = int(math.floor((inter_extent[3] - inter_extent[1]) / out_grid_y)) | ||
327 | - | ||
328 | - # 相交部分在出图的哪个位置 | ||
329 | - | ||
330 | - overview_raster: ndarray = overview.ReadAsArray(off_x, off_y, x_g, y_g, out_x_g, | ||
331 | - out_y_g) | ||
332 | - | ||
333 | - dat = numpy.zeros((256, 256), dtype=int) + 65536 | ||
334 | - dat[out_off_y:out_off_y + out_y_g, out_off_x:out_off_x + out_x_g] = overview_raster | ||
335 | - | ||
336 | - empty = dat | ||
337 | - | ||
338 | - pixel_array[:, :, ceng] = empty | ||
339 | - ceng += 1 | ||
340 | - return pixel_array | ||
341 | - | ||
342 | - def determin_intersect(self,extent1,extent2): | ||
343 | - g1 = GeometryAdapter.envelop_2_polygon(extent1) | ||
344 | - g2 = GeometryAdapter.envelop_2_polygon(extent2) | ||
345 | - return g1.Intersect(g2) | ||
346 | - | ||
347 | - api_doc = { | ||
348 | - "tags": ["影像接口"], | ||
349 | - "parameters": [ | ||
350 | - {"name": "guid", | ||
351 | - "in": "formData", | ||
352 | - "type": "string"}, | ||
353 | - {"name": "tilematrix", | ||
354 | - "in": "formData", | ||
355 | - "type": "string"}, | ||
356 | - {"name": "tilerow", | ||
357 | - "in": "formData", | ||
358 | - "type": "string"}, | ||
359 | - {"name": "tilecol", | ||
360 | - "in": "formData", | ||
361 | - "type": "string"}, | ||
362 | - {"name": "format", | ||
363 | - "in": "formData", | ||
364 | - "type": "string"}, | ||
365 | - {"name": "quality", | ||
366 | - "in": "formData", | ||
367 | - "type": "string"} | ||
368 | - | ||
369 | - ], | ||
370 | - "responses": { | ||
371 | - 200: { | ||
372 | - "schema": { | ||
373 | - "properties": { | ||
374 | - } | ||
375 | - } | ||
376 | - } | ||
377 | - } | ||
378 | - } | ||
379 | - | ||
380 | -class MyThread(Thread): | ||
381 | - def __init__(self,func,args=()): | ||
382 | - super(MyThread,self).__init__() | ||
383 | - self.func = func | ||
384 | - self.args = args | ||
385 | - def run(self): | ||
386 | - self.result = self.func(*self.args) | ||
387 | - def get_result(self): | ||
388 | - try: | ||
389 | - return self.result | ||
390 | - except Exception: | ||
391 | - return None | ||
392 | - | ||
393 | - | ||
394 | - |
app/modules/service/__init__.py
0 → 100644
app/modules/service/address/__init__.py
0 → 100644
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
app/modules/service/image/data/t1/test.TIF
0 → 100644
不能预览此文件类型
@@ -5,14 +5,11 @@ | @@ -5,14 +5,11 @@ | ||
5 | 5 | ||
6 | 6 | ||
7 | from app.util.component.ApiTemplate import ApiTemplate | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | -from thrift.transport import TSocket | ||
9 | -from thrift.transport import TTransport | ||
10 | -from thrift.protocol import TBinaryProtocol | 8 | + |
11 | import json | 9 | import json |
12 | -from app.modules.image.ImageDataService import ImageDataService | ||
13 | from app.util.component.FileProcess import FileProcess | 10 | from app.util.component.FileProcess import FileProcess |
14 | import datetime | 11 | import datetime |
15 | -from app.util.component.ThriftConnect import ThriftConnect | 12 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect |
16 | import os | 13 | import os |
17 | class Api(ApiTemplate): | 14 | class Api(ApiTemplate): |
18 | 15 |
app/modules/service/image/file_tmp/f11036cb-121b-11ec-8b93-f82819efd8e6/Conf(CGCS2000).xml
0 → 100644
1 | +<?xml version="1.0" encoding="utf-8" ?><CacheInfo xsi:type='typens:CacheInfo' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.1'><TileCacheInfo xsi:type='typens:TileCacheInfo'><SpatialReference xsi:type='typens:GeographicCoordinateSystem'><WKT>GEOGCS["GCS_China_Geodetic_Coordinate_System_2000",DATUM["D_China_2000",SPHEROID["CGCS2000",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433],METADATA["China",73.66,7.16,134.85,53.59,0.0,0.0174532925199433,0.0,1067],AUTHORITY["EPSG",4490]]</WKT><XOrigin>-400</XOrigin><YOrigin>-400</YOrigin><XYScale>11258999068426.24</XYScale><ZOrigin>-100000</ZOrigin><ZScale>10000</ZScale><MOrigin>-100000</MOrigin><MScale>10000</MScale><XYTolerance>8.9831528411952133e-009</XYTolerance><ZTolerance>0.001</ZTolerance><MTolerance>0.001</MTolerance><HighPrecision>true</HighPrecision><LeftLongitude>-180</LeftLongitude><WKID>4490</WKID><LatestWKID>4490</LatestWKID></SpatialReference><TileOrigin xsi:type='typens:PointN'><X>-400</X><Y>399.99999999999977</Y></TileOrigin><TileCols>256</TileCols><TileRows>256</TileRows><DPI>96</DPI><PreciseDPI>96</PreciseDPI><LODInfos xsi:type='typens:ArrayOfLODInfo'><LODInfo xsi:type='typens:LODInfo'><LevelID>0</LevelID><Scale>1154287.473</Scale><Resolution>0.0027465820315218724</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>1</LevelID><Scale>577143.73640000005</Scale><Resolution>0.0013732910155229902</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>2</LevelID><Scale>288571.86820000003</Scale><Resolution>0.00068664550776149512</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>3</LevelID><Scale>144285.93410000001</Scale><Resolution>0.00034332275388074756</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>4</LevelID><Scale>72142.967059999995</Scale><Resolution>0.00017166137696416836</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>5</LevelID><Scale>36071.483529999998</Scale><Resolution>8.5830688482084179e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>6</LevelID><Scale>18035.741760000001</Scale><Resolution>4.2915344229144793e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>7</LevelID><Scale>9017.8708819999993</Scale><Resolution>2.1457672119331316e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>8</LevelID><Scale>4508.9354409999996</Scale><Resolution>1.0728836059665658e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>9</LevelID><Scale>2254.4677200000001</Scale><Resolution>5.3644180286430992e-006</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>10</LevelID><Scale>1127.23386</Scale><Resolution>2.6822090143215496e-006</Resolution></LODInfo></LODInfos></TileCacheInfo><TileImageInfo xsi:type='typens:TileImageInfo'><CacheTileFormat>PNG32</CacheTileFormat><CompressionQuality>0</CompressionQuality><Antialiasing>false</Antialiasing></TileImageInfo><CacheStorageInfo xsi:type='typens:CacheStorageInfo'><StorageFormat>esriMapCacheStorageModeCompact</StorageFormat><PacketSize>128</PacketSize></CacheStorageInfo></CacheInfo> |
@@ -7,7 +7,7 @@ | @@ -7,7 +7,7 @@ | ||
7 | from app.util.component.ApiTemplate import ApiTemplate | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | from app.util.component.ModelVisitor import ModelVisitor | 8 | from app.util.component.ModelVisitor import ModelVisitor |
9 | 9 | ||
10 | -from app.modules.image.models import Image | 10 | +from app.modules.service.image.models import Image |
11 | from sqlalchemy import or_,and_ | 11 | from sqlalchemy import or_,and_ |
12 | class Api(ApiTemplate): | 12 | class Api(ApiTemplate): |
13 | 13 |
@@ -7,8 +7,7 @@ | @@ -7,8 +7,7 @@ | ||
7 | from osgeo import gdal,ogr,osr | 7 | from osgeo import gdal,ogr,osr |
8 | from osgeo.gdal import Dataset,Band | 8 | from osgeo.gdal import Dataset,Band |
9 | from app.util.component.ApiTemplate import ApiTemplate | 9 | from app.util.component.ApiTemplate import ApiTemplate |
10 | -from app.util.component.ThriftConnect import ThriftConnect | ||
11 | - | 10 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect |
12 | import json | 11 | import json |
13 | from .models import Image | 12 | from .models import Image |
14 | import datetime | 13 | import datetime |
@@ -6,7 +6,7 @@ | @@ -6,7 +6,7 @@ | ||
6 | 6 | ||
7 | from app.util.component.ApiTemplate import ApiTemplate | 7 | from app.util.component.ApiTemplate import ApiTemplate |
8 | from app.util.component.ModelVisitor import ModelVisitor | 8 | from app.util.component.ModelVisitor import ModelVisitor |
9 | -from app.modules.image.models import ImageService | 9 | +from app.modules.service.image.models import ImageService |
10 | from sqlalchemy import or_ | 10 | from sqlalchemy import or_ |
11 | 11 | ||
12 | class Api(ApiTemplate): | 12 | class Api(ApiTemplate): |
@@ -3,7 +3,7 @@ | @@ -3,7 +3,7 @@ | ||
3 | #createtime: 2021/7/19 | 3 | #createtime: 2021/7/19 |
4 | #email: nheweijun@sina.com | 4 | #email: nheweijun@sina.com |
5 | 5 | ||
6 | -from app.modules.image.models import ImageService,Image | 6 | +from app.modules.service.image.models import ImageService,Image |
7 | from app.models import db | 7 | from app.models import db |
8 | from app.util.component.ApiTemplate import ApiTemplate | 8 | from app.util.component.ApiTemplate import ApiTemplate |
9 | import uuid | 9 | import uuid |
@@ -16,7 +16,7 @@ from PIL import Image | @@ -16,7 +16,7 @@ from PIL import Image | ||
16 | 16 | ||
17 | import time | 17 | import time |
18 | import cv2 | 18 | import cv2 |
19 | -from app.modules.image.models import ImageService,Image | 19 | +from app.modules.service.image.models import ImageService,Image |
20 | from app.models import db | 20 | from app.models import db |
21 | from app.util.component.ApiTemplate import ApiTemplate | 21 | from app.util.component.ApiTemplate import ApiTemplate |
22 | import uuid | 22 | import uuid |
@@ -30,7 +30,7 @@ from kazoo.client import KazooClient | @@ -30,7 +30,7 @@ from kazoo.client import KazooClient | ||
30 | from app import GLOBAL_DIC | 30 | from app import GLOBAL_DIC |
31 | from threading import Thread | 31 | from threading import Thread |
32 | 32 | ||
33 | -from app.util.component.ThriftConnect import ThriftConnect | 33 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect |
34 | from flask import current_app | 34 | from flask import current_app |
35 | import gzip | 35 | import gzip |
36 | import random | 36 | import random |
@@ -13,10 +13,10 @@ from flask import Response | @@ -13,10 +13,10 @@ from flask import Response | ||
13 | import random | 13 | import random |
14 | import time | 14 | import time |
15 | import cv2 | 15 | import cv2 |
16 | -from app.modules.image.models import ImageService | 16 | +from app.modules.service.image.models import ImageService |
17 | from app.util.component.ApiTemplate import ApiTemplate | 17 | from app.util.component.ApiTemplate import ApiTemplate |
18 | from app.util.component.GeometryAdapter import GeometryAdapter | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
19 | -from app.util.component.ThriftConnect import ThriftConnect | 19 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect |
20 | from app.util.component.ParameterUtil import ParameterUtil | 20 | from app.util.component.ParameterUtil import ParameterUtil |
21 | import json | 21 | import json |
22 | from kazoo.client import KazooClient | 22 | from kazoo.client import KazooClient |
@@ -6,7 +6,7 @@ | @@ -6,7 +6,7 @@ | ||
6 | from thrift.transport import TSocket | 6 | from thrift.transport import TSocket |
7 | from thrift.transport import TTransport | 7 | from thrift.transport import TTransport |
8 | from thrift.protocol import TBinaryProtocol | 8 | from thrift.protocol import TBinaryProtocol |
9 | -from app.modules.image.ImageDataService import ImageDataService | 9 | +from app.modules.service.image.ImageDataService import ImageDataService |
10 | 10 | ||
11 | class ThriftConnect: | 11 | class ThriftConnect: |
12 | ''' | 12 | ''' |
app/modules/service/image/util/__init__.py
0 → 100644
app/modules/service/wms/__init__.py
0 → 100644
app/modules/service/wmts/__init__.py
0 → 100644
请
注册
或
登录
后发表评论