正在显示
87 个修改的文件
包含
179 行增加
和
547 行删除
| ... | ... | @@ -3,7 +3,6 @@ import decimal |
| 3 | 3 | from flask import Flask as _Flask |
| 4 | 4 | from flask.json import JSONEncoder as _JSONEncoder |
| 5 | 5 | from flask_cors import CORS |
| 6 | -import time | |
| 7 | 6 | |
| 8 | 7 | from sqlalchemy.sql.expression import true |
| 9 | 8 | import configure |
| ... | ... | @@ -12,24 +11,14 @@ from app.util import find_class |
| 12 | 11 | from app.models import db, Table, InsertingLayerName, Database, DES, Task |
| 13 | 12 | from app.modules.auth.oauth2 import config_oauth, myCodeIDToken |
| 14 | 13 | from flasgger import Swagger |
| 15 | -# from rtree import index | |
| 16 | 14 | import logging |
| 17 | -from sqlalchemy.orm import Session | |
| 18 | -import multiprocessing | |
| 19 | 15 | from app.util.component.EntryData import EntryData |
| 20 | 16 | from app.util.component.EntryDataVacuate import EntryDataVacuate |
| 21 | -import json | |
| 22 | 17 | import threading |
| 23 | -import traceback | |
| 24 | -from sqlalchemy import distinct | |
| 25 | -import uuid | |
| 26 | -from osgeo.ogr import DataSource | |
| 27 | -import datetime | |
| 28 | -from sqlalchemy import or_ | |
| 29 | 18 | from app.util.component.StructuredPrint import StructurePrint |
| 30 | 19 | from app.util.component.PGUtil import PGUtil |
| 31 | 20 | import os |
| 32 | - | |
| 21 | +from app.modules.data.io.data_entry_center import data_entry_center | |
| 33 | 22 | |
| 34 | 23 | class JSONEncoder(_JSONEncoder): |
| 35 | 24 | """ |
| ... | ... | @@ -105,7 +94,6 @@ def create_app(): |
| 105 | 94 | app.register_blueprint(api.bp) |
| 106 | 95 | |
| 107 | 96 | # 入库监测线程 |
| 108 | - | |
| 109 | 97 | @app.before_first_request |
| 110 | 98 | def data_entry_process(): |
| 111 | 99 | StructurePrint.print("start listen") |
| ... | ... | @@ -120,99 +108,3 @@ def create_app(): |
| 120 | 108 | |
| 121 | 109 | return app |
| 122 | 110 | |
| 123 | - | |
| 124 | -def data_entry_center(): | |
| 125 | - running_dict = {} | |
| 126 | - sys_session: Session = PGUtil.get_db_session( | |
| 127 | - configure.SQLALCHEMY_DATABASE_URI) | |
| 128 | - | |
| 129 | - while True: | |
| 130 | - | |
| 131 | - try: | |
| 132 | - time.sleep(3) | |
| 133 | - | |
| 134 | - # 已经结束的进程 从监测中删除 | |
| 135 | - remove_process = [] | |
| 136 | - | |
| 137 | - # structured_print(running_dict.__len__().__str__()) | |
| 138 | - | |
| 139 | - for process, layer_names in running_dict.items(): | |
| 140 | - if not process.is_alive(): | |
| 141 | - for l in layer_names: | |
| 142 | - inserted = sys_session.query( | |
| 143 | - InsertingLayerName).filter_by(name=l).one_or_none() | |
| 144 | - if inserted: | |
| 145 | - sys_session.delete(inserted) | |
| 146 | - sys_session.commit() | |
| 147 | - remove_process.append(process) | |
| 148 | - for process in remove_process: | |
| 149 | - running_dict.pop(process) | |
| 150 | - | |
| 151 | - # StructurePrint.print("listening...") | |
| 152 | - | |
| 153 | - # 入库进程少于阈值,开启入库进程 | |
| 154 | - | |
| 155 | - inter_size = sys_session.query( | |
| 156 | - distinct(InsertingLayerName.task_guid)).count() | |
| 157 | - | |
| 158 | - if inter_size < configure.entry_data_thread: | |
| 159 | - # 锁表啊 | |
| 160 | - ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | |
| 161 | - Task.create_time).with_lockmode("update").limit(1).one_or_none() | |
| 162 | - if ready_task: | |
| 163 | - | |
| 164 | - try: | |
| 165 | - parameter = json.loads(ready_task.parameter) | |
| 166 | - StructurePrint.print("检测到入库任务") | |
| 167 | - ready_task.state = 2 | |
| 168 | - ready_task.process = "入库中" | |
| 169 | - sys_session.commit() | |
| 170 | - | |
| 171 | - metas: list = json.loads( | |
| 172 | - parameter.get("meta").__str__()) | |
| 173 | - parameter["meta"] = metas | |
| 174 | - | |
| 175 | - database = sys_session.query(Database).filter_by( | |
| 176 | - guid=ready_task.database_guid).one_or_none() | |
| 177 | - pg_ds: DataSource = PGUtil.open_pg_data_source( | |
| 178 | - 1, DES.decode(database.sqlalchemy_uri)) | |
| 179 | - | |
| 180 | - this_task_layer = [] | |
| 181 | - for meta in metas: | |
| 182 | - overwrite = parameter.get("overwrite", "no") | |
| 183 | - | |
| 184 | - for layer_name_origin, layer_name in meta.get("layer").items(): | |
| 185 | - origin_name = layer_name | |
| 186 | - no = 1 | |
| 187 | - | |
| 188 | - while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | |
| 189 | - layer_name = origin_name + "_{}".format(no) | |
| 190 | - no += 1 | |
| 191 | - | |
| 192 | - # 添加到正在入库的列表中 | |
| 193 | - iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | |
| 194 | - task_guid=ready_task.guid, | |
| 195 | - name=layer_name) | |
| 196 | - | |
| 197 | - sys_session.add(iln) | |
| 198 | - sys_session.commit() | |
| 199 | - this_task_layer.append(layer_name) | |
| 200 | - # 修改表名 | |
| 201 | - meta["layer"][layer_name_origin] = layer_name | |
| 202 | - | |
| 203 | - pg_ds.Destroy() | |
| 204 | - entry_data_process = multiprocessing.Process( | |
| 205 | - target=EntryDataVacuate().entry, args=(parameter,)) | |
| 206 | - entry_data_process.start() | |
| 207 | - running_dict[entry_data_process] = this_task_layer | |
| 208 | - except Exception as e: | |
| 209 | - sys_session.query(Task).filter_by(guid=ready_task.guid).update( | |
| 210 | - {"state": -1, "process": "入库失败"}) | |
| 211 | - sys_session.commit() | |
| 212 | - StructurePrint.print(e.__str__(), "error") | |
| 213 | - else: | |
| 214 | - # 解表啊 | |
| 215 | - sys_session.commit() | |
| 216 | - except Exception as e: | |
| 217 | - sys_session.commit() | |
| 218 | - StructurePrint.print(e.__str__(), "error") | |
| \ No newline at end of file | ... | ... |
| ... | ... | @@ -34,13 +34,14 @@ class DES(): |
| 34 | 34 | db = SQLAlchemy() |
| 35 | 35 | |
| 36 | 36 | # 动态加载Model |
| 37 | -module_dir = "modules" | |
| 38 | -model_pkg = "models" | |
| 39 | 37 | current_dir = os.path.abspath(os.path.dirname(__file__)) |
| 40 | -for pkg in glob.glob('%s/%s/*/%s' % (current_dir,module_dir,model_pkg)) : | |
| 41 | - module = os.path.basename(os.path.dirname(pkg)) | |
| 42 | - base_name = os.path.basename(pkg).split('.')[0] | |
| 43 | - pkg_name = ".".join(["app.modules",module,base_name]) | |
| 38 | +pkgs = list(glob.glob('%s/modules/*/models' % (current_dir))) | |
| 39 | +pkgs.extend(list(glob.glob('%s/modules/*/*/models' % (current_dir)))) | |
| 40 | + | |
| 41 | +for pkg in pkgs : | |
| 42 | + pkg = os.path.normpath(pkg) | |
| 43 | + node_list = pkg.split(os.path.sep) | |
| 44 | + pkg_name = ".".join(node_list[node_list.index("app"):]) | |
| 44 | 45 | try: |
| 45 | 46 | __import__(pkg_name) |
| 46 | 47 | except: |
| ... | ... | @@ -199,21 +200,21 @@ class InsertingLayerName(db.Model): |
| 199 | 200 | name = Column(String(256)) |
| 200 | 201 | |
| 201 | 202 | |
| 202 | -# class Service(db.Model): | |
| 203 | -# ''' | |
| 204 | -# ''' | |
| 205 | -# __tablename__ = 'dmdms_service' | |
| 206 | -# guid = Column(String(256), primary_key=True) | |
| 207 | -# name=Column(String(256)) | |
| 208 | -# alias = Column(String(256)) | |
| 209 | -# #服务状态 | |
| 210 | -# state= Column(Integer) | |
| 211 | -# create_time = Column(DateTime) | |
| 212 | -# update_time = Column(DateTime) | |
| 213 | -# #服务描述 | |
| 214 | -# description = Column(Text) | |
| 215 | -# #服务节点 | |
| 216 | -# node = Column(Integer) | |
| 217 | -# #服务缩略图 | |
| 218 | -# overview = Column(Binary) | |
| 203 | +class Service(db.Model): | |
| 204 | + ''' | |
| 205 | + ''' | |
| 206 | + __tablename__ = 'dmdms_service' | |
| 207 | + guid = Column(String(256), primary_key=True) | |
| 208 | + name=Column(String(256)) | |
| 209 | + alias = Column(String(256)) | |
| 210 | + #服务状态 | |
| 211 | + state= Column(Integer) | |
| 212 | + create_time = Column(DateTime) | |
| 213 | + update_time = Column(DateTime) | |
| 214 | + #服务描述 | |
| 215 | + description = Column(Text) | |
| 216 | + #服务节点 | |
| 217 | + node = Column(Integer) | |
| 218 | + #服务缩略图 | |
| 219 | + overview = Column(Binary) | |
| 219 | 220 | ... | ... |
app/modules/data/__init__.py
0 → 100644
| ... | ... | @@ -8,7 +8,7 @@ from app.models import Database,db,DES,Table,Columns,TableVacuate |
| 8 | 8 | import uuid |
| 9 | 9 | from . import database_test |
| 10 | 10 | # from app.util import open_pg_data_source |
| 11 | -from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn,Feature,Geometry | |
| 11 | +from osgeo.ogr import DataSource,Layer,FeatureDefn,FieldDefn | |
| 12 | 12 | from sqlalchemy.orm import Session |
| 13 | 13 | from sqlalchemy import create_engine |
| 14 | 14 | from sqlalchemy.orm import sessionmaker |
| ... | ... | @@ -17,7 +17,8 @@ from app.util.component.PGUtil import PGUtil |
| 17 | 17 | from app.util.component.SQLUtil import SQLUtil |
| 18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
| 19 | 19 | from app.util.component.StructuredPrint import StructurePrint |
| 20 | -import configure | |
| 20 | + | |
| 21 | + | |
| 21 | 22 | class Api(ApiTemplate): |
| 22 | 23 | api_name = "注册数据库" |
| 23 | 24 | def process(self): | ... | ... |
| ... | ... | @@ -6,15 +6,13 @@ from flasgger import swag_from |
| 6 | 6 | from flask import Blueprint |
| 7 | 7 | from app.util import BlueprintApi |
| 8 | 8 | |
| 9 | -from flask import send_from_directory,current_app,send_file | |
| 9 | +from flask import send_from_directory | |
| 10 | 10 | import os |
| 11 | -import shutil | |
| 12 | 11 | from . import data_download |
| 13 | 12 | from . import get_meta |
| 14 | 13 | from . import data_entry_by_meta |
| 15 | 14 | from . import get_data_list |
| 16 | -from flask import after_this_request | |
| 17 | -import time | |
| 15 | + | |
| 18 | 16 | |
| 19 | 17 | class DataManager(BlueprintApi): |
| 20 | 18 | ... | ... |
app/modules/data/io/data_entry_center.py
0 → 100644
| 1 | +# coding=utf-8 | |
| 2 | +#author: 4N | |
| 3 | +#createtime: 2021/9/14 | |
| 4 | +#email: nheweijun@sina.com | |
| 5 | + | |
| 6 | +import configure | |
| 7 | +from app.models import InsertingLayerName, Database, DES, Task | |
| 8 | +from sqlalchemy.orm import Session | |
| 9 | +import multiprocessing | |
| 10 | +from app.util.component.EntryDataVacuate import EntryDataVacuate | |
| 11 | +import json | |
| 12 | +from sqlalchemy import distinct | |
| 13 | +import uuid | |
| 14 | +from osgeo.ogr import DataSource | |
| 15 | +from app.util.component.StructuredPrint import StructurePrint | |
| 16 | +from app.util.component.PGUtil import PGUtil | |
| 17 | +import time | |
| 18 | + | |
| 19 | +def data_entry_center(): | |
| 20 | + running_dict = {} | |
| 21 | + sys_session: Session = PGUtil.get_db_session( | |
| 22 | + configure.SQLALCHEMY_DATABASE_URI) | |
| 23 | + | |
| 24 | + while True: | |
| 25 | + | |
| 26 | + try: | |
| 27 | + time.sleep(3) | |
| 28 | + | |
| 29 | + # 已经结束的进程 从监测中删除 | |
| 30 | + remove_process = [] | |
| 31 | + | |
| 32 | + # structured_print(running_dict.__len__().__str__()) | |
| 33 | + | |
| 34 | + for process, layer_names in running_dict.items(): | |
| 35 | + if not process.is_alive(): | |
| 36 | + for l in layer_names: | |
| 37 | + inserted = sys_session.query( | |
| 38 | + InsertingLayerName).filter_by(name=l).one_or_none() | |
| 39 | + if inserted: | |
| 40 | + sys_session.delete(inserted) | |
| 41 | + sys_session.commit() | |
| 42 | + remove_process.append(process) | |
| 43 | + for process in remove_process: | |
| 44 | + running_dict.pop(process) | |
| 45 | + | |
| 46 | + # StructurePrint.print("listening...") | |
| 47 | + | |
| 48 | + # 入库进程少于阈值,开启入库进程 | |
| 49 | + | |
| 50 | + inter_size = sys_session.query( | |
| 51 | + distinct(InsertingLayerName.task_guid)).count() | |
| 52 | + | |
| 53 | + if inter_size < configure.entry_data_thread: | |
| 54 | + # 锁表啊 | |
| 55 | + ready_task: Task = sys_session.query(Task).filter_by(state=0, task_type=1).order_by( | |
| 56 | + Task.create_time).with_lockmode("update").limit(1).one_or_none() | |
| 57 | + if ready_task: | |
| 58 | + | |
| 59 | + try: | |
| 60 | + parameter = json.loads(ready_task.parameter) | |
| 61 | + StructurePrint.print("检测到入库任务") | |
| 62 | + ready_task.state = 2 | |
| 63 | + ready_task.process = "入库中" | |
| 64 | + sys_session.commit() | |
| 65 | + | |
| 66 | + metas: list = json.loads( | |
| 67 | + parameter.get("meta").__str__()) | |
| 68 | + parameter["meta"] = metas | |
| 69 | + | |
| 70 | + database = sys_session.query(Database).filter_by( | |
| 71 | + guid=ready_task.database_guid).one_or_none() | |
| 72 | + pg_ds: DataSource = PGUtil.open_pg_data_source( | |
| 73 | + 1, DES.decode(database.sqlalchemy_uri)) | |
| 74 | + | |
| 75 | + this_task_layer = [] | |
| 76 | + for meta in metas: | |
| 77 | + overwrite = parameter.get("overwrite", "no") | |
| 78 | + | |
| 79 | + for layer_name_origin, layer_name in meta.get("layer").items(): | |
| 80 | + origin_name = layer_name | |
| 81 | + no = 1 | |
| 82 | + | |
| 83 | + while (overwrite.__eq__("no") and pg_ds.GetLayerByName(layer_name)) or sys_session.query(InsertingLayerName).filter_by(name=layer_name).one_or_none(): | |
| 84 | + layer_name = origin_name + "_{}".format(no) | |
| 85 | + no += 1 | |
| 86 | + | |
| 87 | + # 添加到正在入库的列表中 | |
| 88 | + iln = InsertingLayerName(guid=uuid.uuid1().__str__(), | |
| 89 | + task_guid=ready_task.guid, | |
| 90 | + name=layer_name) | |
| 91 | + | |
| 92 | + sys_session.add(iln) | |
| 93 | + sys_session.commit() | |
| 94 | + this_task_layer.append(layer_name) | |
| 95 | + # 修改表名 | |
| 96 | + meta["layer"][layer_name_origin] = layer_name | |
| 97 | + | |
| 98 | + pg_ds.Destroy() | |
| 99 | + entry_data_process = multiprocessing.Process( | |
| 100 | + target=EntryDataVacuate().entry, args=(parameter,)) | |
| 101 | + entry_data_process.start() | |
| 102 | + running_dict[entry_data_process] = this_task_layer | |
| 103 | + except Exception as e: | |
| 104 | + sys_session.query(Task).filter_by(guid=ready_task.guid).update( | |
| 105 | + {"state": -1, "process": "入库失败"}) | |
| 106 | + sys_session.commit() | |
| 107 | + StructurePrint.print(e.__str__(), "error") | |
| 108 | + else: | |
| 109 | + # 解表啊 | |
| 110 | + sys_session.commit() | |
| 111 | + except Exception as e: | |
| 112 | + sys_session.commit() | |
| 113 | + StructurePrint.print(e.__str__(), "error") | |
| \ No newline at end of file | ... | ... |
app/modules/image/image_tilebac.py
deleted
100644 → 0
| 1 | -# coding=utf-8 | |
| 2 | -#author: 4N | |
| 3 | -#createtime: 2021/3/24 | |
| 4 | -#email: nheweijun@sina.com | |
| 5 | - | |
| 6 | -from app.util import * | |
| 7 | -import traceback | |
| 8 | -from osgeo import gdal | |
| 9 | -from osgeo.gdal import * | |
| 10 | -from numpy import ndarray | |
| 11 | -import numpy | |
| 12 | -from flask import Response | |
| 13 | -import io | |
| 14 | -import os | |
| 15 | -from PIL import Image | |
| 16 | - | |
| 17 | -import time | |
| 18 | -import cv2 | |
| 19 | -from app.modules.image.models import ImageService,Image | |
| 20 | -from app.models import db | |
| 21 | -from app.util.component.ApiTemplate import ApiTemplate | |
| 22 | -import uuid | |
| 23 | -from app.util.component.SliceScheme import SliceScheme | |
| 24 | -from app.util.component.FileProcess import FileProcess | |
| 25 | -from app.util.component.ParameterUtil import ParameterUtil | |
| 26 | -from app.util.component.GeometryAdapter import GeometryAdapter | |
| 27 | -import os | |
| 28 | -import json | |
| 29 | -from kazoo.client import KazooClient | |
| 30 | -from app import GLOBAL_DIC | |
| 31 | -from threading import Thread | |
| 32 | -from thrift.transport import TSocket | |
| 33 | -from thrift.transport import TTransport | |
| 34 | -from thrift.protocol import TBinaryProtocol | |
| 35 | -from .ImageDataService import ImageDataService | |
| 36 | -from flask import current_app | |
| 37 | -import gzip | |
| 38 | -import random | |
| 39 | -class Api(ApiTemplate): | |
| 40 | - | |
| 41 | - api_name = "切片" | |
| 42 | - | |
| 43 | - def __init__(self,guid,level, row, col): | |
| 44 | - super().__init__() | |
| 45 | - self.guid = guid | |
| 46 | - self.level = level | |
| 47 | - self.row = row | |
| 48 | - self.col = col | |
| 49 | - | |
| 50 | - def process(self): | |
| 51 | - | |
| 52 | - result = {} | |
| 53 | - parameter: dict = self.para | |
| 54 | - | |
| 55 | - try: | |
| 56 | - if parameter.get("guid"): | |
| 57 | - self.guid = parameter.get("guid") | |
| 58 | - | |
| 59 | - image_service = ImageService.query.filter_by(guid = self.guid).one_or_none() | |
| 60 | - images = image_service.images.all() | |
| 61 | - | |
| 62 | - zoo = GLOBAL_DIC.get("zookeeper") | |
| 63 | - if zoo is None: | |
| 64 | - zoo :KazooClient = KazooClient(hosts=configure.zookeeper, timeout=100) | |
| 65 | - zoo.start() | |
| 66 | - GLOBAL_DIC["zookeeper"] = zoo | |
| 67 | - else : | |
| 68 | - if not zoo.connected: | |
| 69 | - zoo.start() | |
| 70 | - | |
| 71 | - bands = [1, 2, 3] | |
| 72 | - | |
| 73 | - | |
| 74 | - # 转换参数 | |
| 75 | - parameter = ParameterUtil.to_lower(parameter) | |
| 76 | - | |
| 77 | - | |
| 78 | - if parameter.get("tilematrix"): | |
| 79 | - if parameter.get("tilematrix").__contains__(":"): | |
| 80 | - self.level = int(parameter.get("tilematrix").split(":")[-1]) | |
| 81 | - else: | |
| 82 | - self.level = int(parameter.get("tilematrix")) | |
| 83 | - if parameter.get("tilerow"): | |
| 84 | - self.row = int(parameter.get("tilerow")) | |
| 85 | - if parameter.get("tilecol"): | |
| 86 | - self.col = int(parameter.get("tilecol")) | |
| 87 | - | |
| 88 | - image_type = parameter.get("format") if parameter.get("format") else "image/png" | |
| 89 | - quality = int(parameter.get("quality")) if parameter.get("quality") else 30 | |
| 90 | - slice_para = json.loads(image_service.slice_scheme) | |
| 91 | - extent = SliceScheme.get_polygon(slice_para, self.level, self.row, self.col) | |
| 92 | - | |
| 93 | - | |
| 94 | - | |
| 95 | - # 多线程获取分布式数据 | |
| 96 | - | |
| 97 | - intersect_image = [im for im in images if self.determin_intersect(json.loads(im.extent),extent)] | |
| 98 | - | |
| 99 | - pixel_array = numpy.zeros((256, 256,3), dtype=int) | |
| 100 | - | |
| 101 | - if len(intersect_image) > 1: | |
| 102 | - | |
| 103 | - # 结果矩阵 | |
| 104 | - empty_list = [numpy.zeros((256,256), dtype=int) + 65536, | |
| 105 | - numpy.zeros((256,256), dtype=int) + 65536, | |
| 106 | - numpy.zeros((256,256), dtype=int) + 65536] | |
| 107 | - | |
| 108 | - pixel_array = numpy.zeros((256,256,3), dtype=int) | |
| 109 | - thread_list = [] | |
| 110 | - | |
| 111 | - for image in intersect_image: | |
| 112 | - #该影像的服务器,随机选取一个 | |
| 113 | - image_servers = image.server.split(",") | |
| 114 | - indx = int(random.random()*len(image_servers)) | |
| 115 | - image_server = image_servers[indx] | |
| 116 | - | |
| 117 | - thread: MyThread = MyThread(self.get_data, args=(zoo,image_server,image,extent,bands)) | |
| 118 | - thread.start() | |
| 119 | - thread_list.append(thread) | |
| 120 | - | |
| 121 | - | |
| 122 | - for thread in thread_list: | |
| 123 | - thread.join() | |
| 124 | - data = thread.get_result() | |
| 125 | - | |
| 126 | - # 掩膜在中央接口生成,合图 | |
| 127 | - mask = numpy.zeros((256,256), dtype=int) | |
| 128 | - mask2 = numpy.zeros((256,256), dtype=int) | |
| 129 | - jizhun = data[:, :, 0] | |
| 130 | - mask[jizhun == 65536] = 1 | |
| 131 | - mask[jizhun != 65536] = 0 | |
| 132 | - mask2[jizhun == 65536] = 0 | |
| 133 | - mask2[jizhun != 65536] = 1 | |
| 134 | - # 掩膜计算 | |
| 135 | - for i, d in enumerate(empty_list): | |
| 136 | - empty_list[i] = empty_list[i] * mask + data[:, :, i] * mask2 | |
| 137 | - for ii in [0, 1, 2]: | |
| 138 | - # opencv 颜色排序为GBR | |
| 139 | - pixel_array[:, :, 2 - ii] = empty_list[ii] | |
| 140 | - | |
| 141 | - elif len(intersect_image) == 1: | |
| 142 | - | |
| 143 | - # 该影像的服务器,随机选取一个 | |
| 144 | - image = intersect_image[0] | |
| 145 | - image_servers = image.server.split(",") | |
| 146 | - indx = int(random.random() * len(image_servers)) | |
| 147 | - image_server = image_servers[indx] | |
| 148 | - pixel_array_t = self.get_data(zoo, image_server, image, extent, bands) | |
| 149 | - pixel_array = numpy.zeros((256,256, 3), dtype=int) | |
| 150 | - for ii in [0, 1, 2]: | |
| 151 | - # opencv 颜色排序为GBR | |
| 152 | - pixel_array[:, :, 2 - ii] = pixel_array_t[:, :, ii] | |
| 153 | - | |
| 154 | - else: | |
| 155 | - # 结果矩阵 | |
| 156 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) + 65536 | |
| 157 | - | |
| 158 | - #将图片生成在内存中,然后直接返回response | |
| 159 | - im_data = self.create_by_opencv(image_type, pixel_array, quality) | |
| 160 | - | |
| 161 | - return Response(im_data, mimetype=image_type.lower()) | |
| 162 | - | |
| 163 | - except Exception as e: | |
| 164 | - print(traceback.format_exc()) | |
| 165 | - result["state"] = -1 | |
| 166 | - result["message"] = e.__str__() | |
| 167 | - return result | |
| 168 | - | |
| 169 | - | |
| 170 | - def determine_level(self,xysize,origin_extent,extent,max_level): | |
| 171 | - ''' | |
| 172 | - 根据范围判断调用金字塔的哪一层 | |
| 173 | - :param xysize: | |
| 174 | - :param origin_extent: | |
| 175 | - :param extent: | |
| 176 | - :param max_level: | |
| 177 | - :return: | |
| 178 | - ''' | |
| 179 | - x = xysize[0] | |
| 180 | - y = xysize[1] | |
| 181 | - level = -1 | |
| 182 | - pixel = x*y * (((extent[2]-extent[0])*(extent[3]-extent[1]))/((origin_extent[2]-origin_extent[0])*(origin_extent[3]-origin_extent[1]))) | |
| 183 | - while pixel>100000 and level<max_level-1: | |
| 184 | - level+=1 | |
| 185 | - x=x/2 | |
| 186 | - y=y/2 | |
| 187 | - pixel = x * y * (((extent[2] - extent[0]) * (extent[3] - extent[1])) / ( | |
| 188 | - (origin_extent[2] - origin_extent[0]) * (origin_extent[3] - origin_extent[1]))) | |
| 189 | - return level | |
| 190 | - | |
| 191 | - def create_by_opencv(self,image_type, pixel_array, quality): | |
| 192 | - | |
| 193 | - if image_type.__eq__("image/jpeg") or image_type.__eq__("image/jpg"): | |
| 194 | - r, buf = cv2.imencode(".jpg", pixel_array, [cv2.IMWRITE_JPEG_QUALITY, quality]) | |
| 195 | - image_out = buf.tobytes() | |
| 196 | - else: | |
| 197 | - four = numpy.zeros((256, 256), dtype=int) + 255 | |
| 198 | - four[pixel_array[:, :, 0] == 65536] = 0 | |
| 199 | - r, buf = cv2.imencode(".png", numpy.dstack((pixel_array, four))) | |
| 200 | - image_out = buf.tobytes() | |
| 201 | - return image_out | |
| 202 | - | |
| 203 | - def get_data(self,zoo,image_server,image,extent,bands): | |
| 204 | - | |
| 205 | - if image_server.__eq__("本地服务器"): | |
| 206 | - data = self.get_local_data(image, extent, bands) | |
| 207 | - else: | |
| 208 | - ser = image_server | |
| 209 | - if zoo.exists("/rpc/{}".format(ser)): | |
| 210 | - data = self.get_remote_data(image, extent) | |
| 211 | - else: | |
| 212 | - data = numpy.zeros((256, 256, 3), dtype=int) + 65536 | |
| 213 | - return data | |
| 214 | - | |
| 215 | - | |
| 216 | - def get_remote_data(self,image,extent,bands): | |
| 217 | - ''' | |
| 218 | - 通过RPC获取远程数据 | |
| 219 | - :param image: | |
| 220 | - :param extent: | |
| 221 | - :param bands: | |
| 222 | - :return: | |
| 223 | - ''' | |
| 224 | - | |
| 225 | - transport = TSocket.TSocket(image.host, image.port) | |
| 226 | - transport = TTransport.TBufferedTransport(transport) | |
| 227 | - protocol = TBinaryProtocol.TBinaryProtocol(transport) | |
| 228 | - client = ImageDataService.Client(protocol) | |
| 229 | - transport.open() | |
| 230 | - t1 = time.time() | |
| 231 | - data = client.getData(image.path, extent, json.loads(image.extent), bands,256,256) | |
| 232 | - transport.close() | |
| 233 | - current_app.logger.info("time {}".format(time.time() - t1)) | |
| 234 | - | |
| 235 | - data = gzip.decompress(data) | |
| 236 | - data = numpy.frombuffer(data, dtype=int) | |
| 237 | - data = data.reshape((256, 256, 3)) | |
| 238 | - | |
| 239 | - return data | |
| 240 | - | |
| 241 | - def get_local_data(self,image,extent,bands): | |
| 242 | - ''' | |
| 243 | - 获取本地数据 | |
| 244 | - :param image: | |
| 245 | - :param extent: | |
| 246 | - :param bands: | |
| 247 | - :return: | |
| 248 | - ''' | |
| 249 | - pixel_array = numpy.zeros((256, 256, 3), dtype=int) | |
| 250 | - ceng = 0 | |
| 251 | - img: Dataset = gdal.Open(image.path, 0) | |
| 252 | - t1 = time.time() | |
| 253 | - for band in bands: | |
| 254 | - | |
| 255 | - # 自决定金字塔等级 | |
| 256 | - xysize = [img.RasterXSize, img.RasterYSize] | |
| 257 | - origin_extent = image.extent | |
| 258 | - band_data: Band = img.GetRasterBand(band) | |
| 259 | - max_level = band_data.GetOverviewCount() | |
| 260 | - | |
| 261 | - # 超出空间范围 | |
| 262 | - if extent[2] < origin_extent[0] or extent[0] > origin_extent[2] or extent[1] > origin_extent[ | |
| 263 | - 3] or extent[3] < origin_extent[1]: | |
| 264 | - empty = numpy.zeros((256, 256), dtype=int) + 65536 | |
| 265 | - # 空间范围相交 | |
| 266 | - else: | |
| 267 | - image_level = self.determine_level(xysize, origin_extent, extent, max_level) | |
| 268 | - | |
| 269 | - if image_level == -1: | |
| 270 | - overview = band_data | |
| 271 | - else: | |
| 272 | - try: | |
| 273 | - overview: Band = band_data.GetOverview(image_level) | |
| 274 | - except: | |
| 275 | - raise Exception("该影像不存在该级别的金字塔数据!") | |
| 276 | - ox = overview.XSize | |
| 277 | - oy = overview.YSize | |
| 278 | - | |
| 279 | - # 网格大小 | |
| 280 | - grid_x = (origin_extent[2] - origin_extent[0]) / (ox * 1.0) | |
| 281 | - grid_y = (origin_extent[3] - origin_extent[1]) / (oy * 1.0) | |
| 282 | - | |
| 283 | - # 完全在影像范围内 | |
| 284 | - if extent[0] > origin_extent[0] and extent[1] > origin_extent[1] and extent[2] < \ | |
| 285 | - origin_extent[2] and extent[3] < origin_extent[3]: | |
| 286 | - | |
| 287 | - # 网格偏移量 | |
| 288 | - off_x = math.floor((extent[0] - origin_extent[0]) / grid_x) | |
| 289 | - off_y = math.floor((origin_extent[3] - extent[3]) / grid_y) | |
| 290 | - | |
| 291 | - # 截取后网格个数 | |
| 292 | - x_g = math.ceil((extent[2] - extent[0]) / grid_x) | |
| 293 | - | |
| 294 | - y_g = math.ceil((extent[3] - extent[1]) / grid_y) | |
| 295 | - | |
| 296 | - empty = overview.ReadAsArray(off_x, off_y, x_g, y_g, 256, 256) | |
| 297 | - | |
| 298 | - | |
| 299 | - # 部分相交 | |
| 300 | - else: | |
| 301 | - | |
| 302 | - inter_extent = [0, 0, 0, 0] | |
| 303 | - inter_extent[0] = origin_extent[0] if origin_extent[0] > extent[0] else extent[0] | |
| 304 | - inter_extent[1] = origin_extent[1] if origin_extent[1] > extent[1] else extent[1] | |
| 305 | - inter_extent[2] = origin_extent[2] if origin_extent[2] < extent[2] else extent[2] | |
| 306 | - inter_extent[3] = origin_extent[3] if origin_extent[3] < extent[3] else extent[3] | |
| 307 | - | |
| 308 | - # 网格偏移量 | |
| 309 | - off_x = math.floor((inter_extent[0] - origin_extent[0]) / grid_x) | |
| 310 | - off_y = math.floor((origin_extent[3] - inter_extent[3]) / grid_y) | |
| 311 | - | |
| 312 | - # 截取后网格个数 | |
| 313 | - x_g = math.floor((inter_extent[2] - inter_extent[0]) / grid_x) | |
| 314 | - y_g = math.floor((inter_extent[3] - inter_extent[1]) / grid_y) | |
| 315 | - | |
| 316 | - # 相对于出图的偏移量 | |
| 317 | - | |
| 318 | - # 出图的网格大小 | |
| 319 | - out_grid_x = (extent[2] - extent[0]) / (256 * 1.0) | |
| 320 | - out_grid_y = (extent[3] - extent[1]) / (256 * 1.0) | |
| 321 | - | |
| 322 | - out_off_x = int(math.ceil((inter_extent[0] - extent[0]) / out_grid_x)) | |
| 323 | - out_off_y = int(math.ceil((extent[3] - inter_extent[3]) / out_grid_y)) | |
| 324 | - | |
| 325 | - out_x_g = int(math.floor((inter_extent[2] - inter_extent[0]) / out_grid_x)) | |
| 326 | - out_y_g = int(math.floor((inter_extent[3] - inter_extent[1]) / out_grid_y)) | |
| 327 | - | |
| 328 | - # 相交部分在出图的哪个位置 | |
| 329 | - | |
| 330 | - overview_raster: ndarray = overview.ReadAsArray(off_x, off_y, x_g, y_g, out_x_g, | |
| 331 | - out_y_g) | |
| 332 | - | |
| 333 | - dat = numpy.zeros((256, 256), dtype=int) + 65536 | |
| 334 | - dat[out_off_y:out_off_y + out_y_g, out_off_x:out_off_x + out_x_g] = overview_raster | |
| 335 | - | |
| 336 | - empty = dat | |
| 337 | - | |
| 338 | - pixel_array[:, :, ceng] = empty | |
| 339 | - ceng += 1 | |
| 340 | - return pixel_array | |
| 341 | - | |
| 342 | - def determin_intersect(self,extent1,extent2): | |
| 343 | - g1 = GeometryAdapter.envelop_2_polygon(extent1) | |
| 344 | - g2 = GeometryAdapter.envelop_2_polygon(extent2) | |
| 345 | - return g1.Intersect(g2) | |
| 346 | - | |
| 347 | - api_doc = { | |
| 348 | - "tags": ["影像接口"], | |
| 349 | - "parameters": [ | |
| 350 | - {"name": "guid", | |
| 351 | - "in": "formData", | |
| 352 | - "type": "string"}, | |
| 353 | - {"name": "tilematrix", | |
| 354 | - "in": "formData", | |
| 355 | - "type": "string"}, | |
| 356 | - {"name": "tilerow", | |
| 357 | - "in": "formData", | |
| 358 | - "type": "string"}, | |
| 359 | - {"name": "tilecol", | |
| 360 | - "in": "formData", | |
| 361 | - "type": "string"}, | |
| 362 | - {"name": "format", | |
| 363 | - "in": "formData", | |
| 364 | - "type": "string"}, | |
| 365 | - {"name": "quality", | |
| 366 | - "in": "formData", | |
| 367 | - "type": "string"} | |
| 368 | - | |
| 369 | - ], | |
| 370 | - "responses": { | |
| 371 | - 200: { | |
| 372 | - "schema": { | |
| 373 | - "properties": { | |
| 374 | - } | |
| 375 | - } | |
| 376 | - } | |
| 377 | - } | |
| 378 | - } | |
| 379 | - | |
| 380 | -class MyThread(Thread): | |
| 381 | - def __init__(self,func,args=()): | |
| 382 | - super(MyThread,self).__init__() | |
| 383 | - self.func = func | |
| 384 | - self.args = args | |
| 385 | - def run(self): | |
| 386 | - self.result = self.func(*self.args) | |
| 387 | - def get_result(self): | |
| 388 | - try: | |
| 389 | - return self.result | |
| 390 | - except Exception: | |
| 391 | - return None | |
| 392 | - | |
| 393 | - | |
| 394 | - |
app/modules/service/__init__.py
0 → 100644
app/modules/service/address/__init__.py
0 → 100644
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
不能预览此文件类型
app/modules/service/image/data/t1/test.TIF
0 → 100644
不能预览此文件类型
| ... | ... | @@ -5,14 +5,11 @@ |
| 5 | 5 | |
| 6 | 6 | |
| 7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
| 8 | -from thrift.transport import TSocket | |
| 9 | -from thrift.transport import TTransport | |
| 10 | -from thrift.protocol import TBinaryProtocol | |
| 8 | + | |
| 11 | 9 | import json |
| 12 | -from app.modules.image.ImageDataService import ImageDataService | |
| 13 | 10 | from app.util.component.FileProcess import FileProcess |
| 14 | 11 | import datetime |
| 15 | -from app.util.component.ThriftConnect import ThriftConnect | |
| 12 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
| 16 | 13 | import os |
| 17 | 14 | class Api(ApiTemplate): |
| 18 | 15 | ... | ... |
app/modules/service/image/file_tmp/f11036cb-121b-11ec-8b93-f82819efd8e6/Conf(CGCS2000).xml
0 → 100644
| 1 | +<?xml version="1.0" encoding="utf-8" ?><CacheInfo xsi:type='typens:CacheInfo' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.1'><TileCacheInfo xsi:type='typens:TileCacheInfo'><SpatialReference xsi:type='typens:GeographicCoordinateSystem'><WKT>GEOGCS["GCS_China_Geodetic_Coordinate_System_2000",DATUM["D_China_2000",SPHEROID["CGCS2000",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433],METADATA["China",73.66,7.16,134.85,53.59,0.0,0.0174532925199433,0.0,1067],AUTHORITY["EPSG",4490]]</WKT><XOrigin>-400</XOrigin><YOrigin>-400</YOrigin><XYScale>11258999068426.24</XYScale><ZOrigin>-100000</ZOrigin><ZScale>10000</ZScale><MOrigin>-100000</MOrigin><MScale>10000</MScale><XYTolerance>8.9831528411952133e-009</XYTolerance><ZTolerance>0.001</ZTolerance><MTolerance>0.001</MTolerance><HighPrecision>true</HighPrecision><LeftLongitude>-180</LeftLongitude><WKID>4490</WKID><LatestWKID>4490</LatestWKID></SpatialReference><TileOrigin xsi:type='typens:PointN'><X>-400</X><Y>399.99999999999977</Y></TileOrigin><TileCols>256</TileCols><TileRows>256</TileRows><DPI>96</DPI><PreciseDPI>96</PreciseDPI><LODInfos xsi:type='typens:ArrayOfLODInfo'><LODInfo xsi:type='typens:LODInfo'><LevelID>0</LevelID><Scale>1154287.473</Scale><Resolution>0.0027465820315218724</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>1</LevelID><Scale>577143.73640000005</Scale><Resolution>0.0013732910155229902</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>2</LevelID><Scale>288571.86820000003</Scale><Resolution>0.00068664550776149512</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>3</LevelID><Scale>144285.93410000001</Scale><Resolution>0.00034332275388074756</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>4</LevelID><Scale>72142.967059999995</Scale><Resolution>0.00017166137696416836</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>5</LevelID><Scale>36071.483529999998</Scale><Resolution>8.5830688482084179e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>6</LevelID><Scale>18035.741760000001</Scale><Resolution>4.2915344229144793e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>7</LevelID><Scale>9017.8708819999993</Scale><Resolution>2.1457672119331316e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>8</LevelID><Scale>4508.9354409999996</Scale><Resolution>1.0728836059665658e-005</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>9</LevelID><Scale>2254.4677200000001</Scale><Resolution>5.3644180286430992e-006</Resolution></LODInfo><LODInfo xsi:type='typens:LODInfo'><LevelID>10</LevelID><Scale>1127.23386</Scale><Resolution>2.6822090143215496e-006</Resolution></LODInfo></LODInfos></TileCacheInfo><TileImageInfo xsi:type='typens:TileImageInfo'><CacheTileFormat>PNG32</CacheTileFormat><CompressionQuality>0</CompressionQuality><Antialiasing>false</Antialiasing></TileImageInfo><CacheStorageInfo xsi:type='typens:CacheStorageInfo'><StorageFormat>esriMapCacheStorageModeCompact</StorageFormat><PacketSize>128</PacketSize></CacheStorageInfo></CacheInfo> | |
| \ No newline at end of file | ... | ... |
| ... | ... | @@ -7,7 +7,7 @@ |
| 7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
| 8 | 8 | from app.util.component.ModelVisitor import ModelVisitor |
| 9 | 9 | |
| 10 | -from app.modules.image.models import Image | |
| 10 | +from app.modules.service.image.models import Image | |
| 11 | 11 | from sqlalchemy import or_,and_ |
| 12 | 12 | class Api(ApiTemplate): |
| 13 | 13 | ... | ... |
| ... | ... | @@ -7,8 +7,7 @@ |
| 7 | 7 | from osgeo import gdal,ogr,osr |
| 8 | 8 | from osgeo.gdal import Dataset,Band |
| 9 | 9 | from app.util.component.ApiTemplate import ApiTemplate |
| 10 | -from app.util.component.ThriftConnect import ThriftConnect | |
| 11 | - | |
| 10 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
| 12 | 11 | import json |
| 13 | 12 | from .models import Image |
| 14 | 13 | import datetime | ... | ... |
| ... | ... | @@ -6,7 +6,7 @@ |
| 6 | 6 | |
| 7 | 7 | from app.util.component.ApiTemplate import ApiTemplate |
| 8 | 8 | from app.util.component.ModelVisitor import ModelVisitor |
| 9 | -from app.modules.image.models import ImageService | |
| 9 | +from app.modules.service.image.models import ImageService | |
| 10 | 10 | from sqlalchemy import or_ |
| 11 | 11 | |
| 12 | 12 | class Api(ApiTemplate): | ... | ... |
| ... | ... | @@ -3,7 +3,7 @@ |
| 3 | 3 | #createtime: 2021/7/19 |
| 4 | 4 | #email: nheweijun@sina.com |
| 5 | 5 | |
| 6 | -from app.modules.image.models import ImageService,Image | |
| 6 | +from app.modules.service.image.models import ImageService,Image | |
| 7 | 7 | from app.models import db |
| 8 | 8 | from app.util.component.ApiTemplate import ApiTemplate |
| 9 | 9 | import uuid | ... | ... |
| ... | ... | @@ -16,7 +16,7 @@ from PIL import Image |
| 16 | 16 | |
| 17 | 17 | import time |
| 18 | 18 | import cv2 |
| 19 | -from app.modules.image.models import ImageService,Image | |
| 19 | +from app.modules.service.image.models import ImageService,Image | |
| 20 | 20 | from app.models import db |
| 21 | 21 | from app.util.component.ApiTemplate import ApiTemplate |
| 22 | 22 | import uuid |
| ... | ... | @@ -30,7 +30,7 @@ from kazoo.client import KazooClient |
| 30 | 30 | from app import GLOBAL_DIC |
| 31 | 31 | from threading import Thread |
| 32 | 32 | |
| 33 | -from app.util.component.ThriftConnect import ThriftConnect | |
| 33 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
| 34 | 34 | from flask import current_app |
| 35 | 35 | import gzip |
| 36 | 36 | import random | ... | ... |
| ... | ... | @@ -13,10 +13,10 @@ from flask import Response |
| 13 | 13 | import random |
| 14 | 14 | import time |
| 15 | 15 | import cv2 |
| 16 | -from app.modules.image.models import ImageService | |
| 16 | +from app.modules.service.image.models import ImageService | |
| 17 | 17 | from app.util.component.ApiTemplate import ApiTemplate |
| 18 | 18 | from app.util.component.GeometryAdapter import GeometryAdapter |
| 19 | -from app.util.component.ThriftConnect import ThriftConnect | |
| 19 | +from app.modules.service.image.util.ThriftConnect import ThriftConnect | |
| 20 | 20 | from app.util.component.ParameterUtil import ParameterUtil |
| 21 | 21 | import json |
| 22 | 22 | from kazoo.client import KazooClient | ... | ... |
| ... | ... | @@ -6,7 +6,7 @@ |
| 6 | 6 | from thrift.transport import TSocket |
| 7 | 7 | from thrift.transport import TTransport |
| 8 | 8 | from thrift.protocol import TBinaryProtocol |
| 9 | -from app.modules.image.ImageDataService import ImageDataService | |
| 9 | +from app.modules.service.image.ImageDataService import ImageDataService | |
| 10 | 10 | |
| 11 | 11 | class ThriftConnect: |
| 12 | 12 | ''' | ... | ... |
app/modules/service/image/util/__init__.py
0 → 100644
app/modules/service/wms/__init__.py
0 → 100644
app/modules/service/wmts/__init__.py
0 → 100644
请
注册
或
登录
后发表评论