diff --git a/doc/CivitFetchPaseudocode b/doc/CivitFetchPaseudocode new file mode 100644 index 0000000..837170a --- /dev/null +++ b/doc/CivitFetchPaseudocode @@ -0,0 +1,115 @@ +# Simple json request module + +aliases: + retry = "retry 10 times with 1 time(s) cooldown" + +start +try get request + +-- Network errors block +Having: url + +Exception: Network is unreachable or temporary failure in name resolution + Wait until network became available +Exception: Name not resolved + Repeat 10 times with 10 times cooldown + Fatal: target site is dead + +-- HTTP errors block +Having: Some HTTP response + +Exception: Service unavailable + Repeat 10 times with 10 times cooldown + Throw Exception on higher level +Exception: Internal server error and other HTTP errors (403, 404...) + retry + Throw Exception on higher level + +-- Content errors block +Having +Some successful HTTP response + +Raised: Service unavailable + wait until initial page become available + retry + try strip cursor if cursor crawler + retry + try decrement cursor/page + retry + try increment cursor/page + retry + +Raised: Internal server error and other HTTP errors (403, 404...) + try strip cursor if cursor crawler + retry + try decrement cursor/page + retry + try increment cursor/page + retry + +Exception: Response is not json data + retry + try strip cursor and retry if cursor crawler + try decrement cursor/page and retry 1 times + try increment cursor/page and retry 1 times + log error and end crawl + +Having: Some json data + +Exception: Response not contains {items: list, metadata: dict} fields + retry + try strip cursor and retry if cursor crawler + try decrement cursor/page and retry 1 times + try increment cursor/page and retry 1 times + log error and end crawl + +Exception: items is empty and metadata is empty + retry + try strip cursor and retry if cursor crawler + try decrement cursor/page and retry 1 times + try increment cursor/page and retry 1 times + log error and end crawl + +Exception: items is empty and metadata is not empty + if result of (try decrement cursor/page and retry 1 times) is 1: end crawl + retry + try strip cursor and retry if cursor crawler + try decrement cursor/page and retry 1 times + log error and end crawl + +Exception: if cursor crawler: metadata not have required field "nextPage" + retry + try strip cursor and retry if cursor crawler + try decrement cursor/page and retry 1 times + try increment cursor/page and retry 1 times + log error and end crawl + +ExitPoint: items is not empty and metadata is empty + end crawl + + +Having: Some valid json api response (items is not empty and not cursor crawler or (end crawl flag is set or metadata is not empty)) + +Exception: Cursor slip (nextPage url equals request url) + if not cursor crawler: pass (not possible) + try increment cursor/page and retry 1 times + + +Exception: response "items" has no new items (may be caused by cursor system destroy or rare situation, where total_items mod page_items_limit is 0) + try strip cursor and retry if cursor crawler + try increment cursor/page and retry 1 times + log error and end crawl + +Warning: Added items != page_items_limit and not end crawl + log warning + +Having: some items, added to all crawl items dict + + + + + + + + + diff --git a/gui.py b/gui.py new file mode 100644 index 0000000..e69de29 diff --git a/modelspace/ModelPackageSubRepository.py b/modelspace/ModelPackageSubRepository.py index 6ae8b80..9de429f 100644 --- a/modelspace/ModelPackageSubRepository.py +++ b/modelspace/ModelPackageSubRepository.py @@ -195,14 +195,14 @@ class ModelPackageSubRepository: 'provides': ic_provides, 'version_id': ic_version_id, 'file_id': ic_file_id, - 'package_type': ic_package_type, + 'package_type': ic_package_type.lower(), 'tags': ic_tags, 'version': ic_version, 'release_date': ic_release_date, 'lineage': ic_lineage, - 'images': ic_images, + 'images': ic_images or list(), 'size_bytes': ic_size_bytes, - 'quantisation': ic_quantisation, + 'quantisation': ic_quantisation or '', 'url': ic_url, 'filename': ic_filename, 'model_info': ic_model_info, @@ -251,14 +251,15 @@ class ModelPackageSubRepository: for i in range(len(available_to_pull)): candidate = available_to_pull[i] + quantisation = candidate['quantisation'] or 'N/A' print(f' {i:<{2}} {candidate['version']:<{10}} {candidate['package_type']:<{10}} {candidate['release_date']:<{25}}' - f' {candidate['lineage']:<{10}} {candidate['quantisation']:<{5}} {format_bytes(candidate['size_bytes']):<{10}} ') + f' {candidate['lineage']:<{10}} {quantisation:<{5}} {format_bytes(candidate['size_bytes']):<{10}} ') if len(available_to_pull) > 1: to_pull = select_elements(pull_candidates, input("Your choice: ")) else: to_pull = available_to_pull # Ввод зависимостей - print("Зависимости (введите по одной, пустая строка для завершения):") + print("Зависимости (введите по одной, п1658427устая строка для завершения):") additional_dependencies = [] while True: dep = input().strip() @@ -351,6 +352,7 @@ class ModelPackageSubRepository: package = ModelPackage(package_path, [], package_info) self.packages[package.uuid] = package + self.package_names.add(package.name) self.add_package_to_collection(package.name, 'civit', internal=True) if collection: self.add_package_to_collection(package.name, collection, category, internal=internal) diff --git a/modules/civit/Civit.py b/modules/civit/Civit.py index aeed5c1..65064e4 100644 --- a/modules/civit/Civit.py +++ b/modules/civit/Civit.py @@ -1,15 +1,76 @@ +import datetime +import json +import os +from pathlib import Path + +from modules.civit.client import Client from modules.civit.fetch import Fetch from modules.civit.datamodel import * from modules.shared.DatabaseAbstraction import Database class Civit: - def __init__(self, db: Database, path): + def __init__(self, db: Database, path, api_key = None): self._db = db self.path = path - self.fetcher = Fetch() + self.client = Client(path, api_key) + self.fetcher = Fetch(self.client) Creator.create(self._db.cursor()) Tag.create(self._db.cursor()) + Model.create(self._db.cursor()) + + def save(self, e: DataClassDatabase): return e.save(self._db.cursor()) + + def from_fetch(self, entity: str, entity_type: type[DataClassDatabase]): + if entity: entity = entity.lower() + else: return + if entity in self.fetcher.entities: subdir = self.fetcher.entities[entity] + else: raise ValueError(f'Civit doesn\'t have entity type {entity}') + directory_path = str(Path(self.client.path) / subdir) + files = os.listdir(directory_path) + i = 0 + files_count = len(files) + tp = datetime.datetime.now() + + # Проходим по всем файлам в директории + for filename in files: + i += 1 + print(f'processing file {i} of {files_count} ({float(i) / float(files_count) * 100:.2f}%): {filename} Elapsed time {datetime.datetime.now() - tp}') + tp = datetime.datetime.now() + if not filename.endswith('.json'): continue + file_path = os.path.join(directory_path, filename) + data = None + + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # Если данные - список словарей + if isinstance(data, list): + pass + # Если данные - один словарь + elif isinstance(data, dict): + data = [data] + + except (json.JSONDecodeError, IOError) as e: + print(f"Ошибка чтения файла {filename}: {e}") + continue + + if not data: continue + + t = datetime.datetime.now() + j = 0 + data_count = len(data) + for d in data: + j += 1 + self.save(entity_type.from_dict(d)) + if j % 1000 == 0: + print(f'saved {j} {entity} of {data_count} ({float(j) / float(data_count) * 100:.2f}%). Elapsed time {datetime.datetime.now() - t}') + t = datetime.datetime.now() + del d, data + + + + + - def creator_save(self, c: Creator): return c.save(self._db.cursor()) - def tag_save(self, t: Tag): return t.save(self._db.cursor()) diff --git a/modules/civit/client.py b/modules/civit/client.py index aae59c6..99ff374 100644 --- a/modules/civit/client.py +++ b/modules/civit/client.py @@ -16,7 +16,7 @@ class ClientConfig(Config): class Client: - def __init__(self, path, api_key: str): + def __init__(self, path, api_key: str = None): self.path = path os.makedirs(self.path, exist_ok=True) diff --git a/modules/civit/datamodel.py b/modules/civit/datamodel.py index 6363f2f..b7f320e 100644 --- a/modules/civit/datamodel.py +++ b/modules/civit/datamodel.py @@ -5,17 +5,203 @@ from modules.shared.DataClassDatabase import DataClassDatabase @dataclass -class Creator(DataClassDatabase): +class ModelVersionFileHashes(DataClassDatabase): + # primitives + AutoV1: Optional[str] = None + AutoV2: Optional[str] = None + AutoV3: Optional[str] = None + CRC32: Optional[str] = None + SHA256: Optional[str] = None + BLAKE3: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._table_name = 'model_versions_files_hashes' + self._standalone_entity = False + +@dataclass +class ModelVersionFileMetadata(DataClassDatabase): + # primitives + format: Optional[str] = None + size: Optional[str] = None + fp: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._table_name = 'model_versions_files_metadata' + self._standalone_entity = False + +@dataclass +class ModelVersionFile(DataClassDatabase): + # primary key + id: Optional[int] = None + # primitives + sizeKB: Optional[float] = None + name: Optional[str] = None + type: Optional[str] = None + pickleScanResult: Optional[str] = None + pickleScanMessage: Optional[str] = None + virusScanResult: Optional[str] = None + virusScanMessage: Optional[str] = None + scannedAt: Optional[str] = None + downloadUrl: Optional[str] = None + primary: Optional[bool] = None + # child entities + metadata: Optional[ModelVersionFileMetadata] = None + hashes: Optional[ModelVersionFileHashes] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = { + 'metadata': ModelVersionFileMetadata, + 'hashes': ModelVersionFileHashes, + } + self._key_field = 'id' + self._table_name = 'model_versions_files' + self._standalone_entity = False + +@dataclass +class ModelVersionStats(DataClassDatabase): + downloadCount: Optional[int] = None + ratingCount: Optional[int] = None + rating: Optional[float] = None + thumbsUpCount: Optional[int] = None + thumbsDownCount: Optional[int] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._table_name = 'model_versions_stats' + self._standalone_entity = False + +@dataclass +class ModelVersionImage(DataClassDatabase): + # primary key + id: Optional[int] = None + # primitives + url: Optional[str] = None + nsfwLevel: Optional[int] = None + width: Optional[int] = None + height: Optional[int] = None + hash: Optional[str] = None + type: Optional[str] = None + minor: Optional[bool] = None + poi: Optional[bool] = None + hasMeta: Optional[bool] = None + hasPositivePrompt: Optional[bool] = None + onSite: Optional[int] = None + remixOfId: Optional[int] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._key_field = 'id' + self._table_name = 'model_versions_images' + self._standalone_entity = False + +@dataclass +class ModelVersion(DataClassDatabase): + # primary key + id: Optional[int] = None + # primitives + index: Optional[int] = None + name: Optional[str] = None + baseModel: Optional[str] = None + baseModelType: Optional[str] = None + publishedAt: Optional[str] = None + availability: Optional[str] = None + nsfwLevel: Optional[int] = None + description: Optional[str] = None + supportsGeneration: Optional[bool] = None + downloadUrl: Optional[str] = None + # list of primitives + trainedWords: Optional[list] = None + # child entities + stats: Optional[ModelVersionStats] = None + files: Optional[list[ModelVersionFile]] = None + images: Optional[list[ModelVersionImage]] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = { + 'stats': ModelVersionStats, + 'files': ModelVersionFile, + 'images': ModelVersionImage, + } + self._key_field = 'id' + self._table_name = 'model_versions' + self._standalone_entity = False + +@dataclass +class ModelStats(DataClassDatabase): + # primitives + downloadCount: Optional[int] = None + favoriteCount: Optional[int] = None + thumbsUpCount: Optional[int] = None + thumbsDownCount: Optional[int] = None + commentCount: Optional[int] = None + ratingCount: Optional[int] = None + rating: Optional[int] = None + tippedAmountCount: Optional[int] = None + + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._table_name = 'model_stats' + self._standalone_entity = False + +@dataclass +class ModelCreator(DataClassDatabase): + # primitives username: Optional[str] = None - modelCount: Optional[int] = None - link: Optional[str] = None image: Optional[str] = None def __post_init__(self): super().__post_init__() self._forwarding = {} - self._key_field = 'username' - self._table_name = 'creators' + self._table_name = 'model_creators' + self._standalone_entity = False + +@dataclass +class Model(DataClassDatabase): + # primary key + id: Optional[int] = None + # primitives + name: Optional[str] = None + description: Optional[str] = None + allowNoCredit: Optional[bool] = None + allowCommercialUse: Optional[list] = None + allowDerivatives: Optional[bool] = None + allowDifferentLicense: Optional[bool] = None + type: Optional[str] = None + minor: Optional[bool] = None + sfwOnly: Optional[bool] = None + poi: Optional[bool] = None + nsfw: Optional[bool] = None + nsfwLevel: Optional[int] = None + availability: Optional[str] = None + cosmetic: Optional[str] = None + supportsGeneration: Optional[bool] = None + mode: Optional[str] = None + # list of primitives + tags: Optional[list] = None + # child entities + stats: Optional[ModelStats] = None + creator: Optional[ModelCreator] = None + modelVersions: Optional[list[ModelVersion]] = None + + + def __post_init__(self): + super().__post_init__() + self._forwarding = { + 'stats': ModelStats, + 'creator': ModelCreator, + 'modelVersions': ModelVersion, + } + self._key_field = 'id' + self._table_name = 'models' self._standalone_entity = True @dataclass @@ -30,99 +216,18 @@ class Tag(DataClassDatabase): self._table_name = 'tags' self._standalone_entity = True +@dataclass +class Creator(DataClassDatabase): + # primary key + username: Optional[str] = None + # primitives + modelCount: Optional[int] = None + link: Optional[str] = None + image: Optional[str] = None -# @dataclass -# class ModelVersionStats(ForwardingBase): -# downloadCount: Optional[int] = None -# ratingCount: Optional[int] = None -# rating: Optional[int] = None -# thumbsUpCount: Optional[int] = None -# thumbsDownCount: Optional[int] = None -# -# def __post_init__(self): -# super().__post_init__() -# self._forwarding = {} -# -# -# -# @dataclass -# class ModelVersion(ForwardingBase): -# id: Optional[int] = None -# index: Optional[int] = None -# name: Optional[str] = None -# baseModel: Optional[str] = None -# baseModelType: Optional[str] = None -# publishedAt: Optional[str] = None -# availability: Optional[str] = None -# nsfwLevel: Optional[int] = None -# description: Optional[str] = None -# trainedWords: Optional[list[str]] = None -# stats: Optional[ModelVersionStats] = None -# supportsGeneration: Optional[bool] = None -# downloadUrl: Optional[str] = None -# # FILES -# # IMAGES -# -# -# def __post_init__(self): -# super().__post_init__() -# self._forwarding = { -# 'stats': ModelVersionStats, -# } -# self._key_field = 'id' -# -# -# -# -# @dataclass -# class ModelStats(ForwardingBase): -# downloadCount: Optional[int] = None -# favoriteCount: Optional[int] = None -# thumbsUpCount: Optional[int] = None -# thumbsDownCount: Optional[int] = None -# commentCount: Optional[int] = None -# ratingCount: Optional[int] = None -# rating: Optional[int] = None -# -# def __post_init__(self): -# super().__post_init__() -# self._forwarding = {} -# -# -# -# -# @dataclass -# class Model(ForwardingBase): -# id: Optional[int] = None -# name: Optional[str] = None -# description: Optional[str] = None -# allowNoCredit: Optional[bool] = None -# allowCommercialUse: Optional[list] = None -# allowDerivatives: Optional[bool] = None -# allowDifferentLicense: Optional[bool] = None -# type: Optional[str] = None -# minor: Optional[bool] = None -# sfwOnly: Optional[bool] = None -# poi: Optional[bool] = None -# nsfw: Optional[bool] = None -# nsfwLevel: Optional[int] = None -# availability: Optional[str] = None -# cosmetic: Optional[str] = None -# supportsGeneration: Optional[bool] = None -# stats: Optional[ModelStats] = None -# creator: Optional[Creator] = None -# tags: Optional[list[Tag]] = None -# modelVersions: Optional[list[ModelVersion]] = None -# -# -# def __post_init__(self): -# super().__post_init__() -# self._forwarding = { -# 'stats': ModelStats, -# 'creator': Creator, -# 'tags': Tag, -# 'modelVersions': ModelVersion, -# } -# self._key_field = 'id' -# - + def __post_init__(self): + super().__post_init__() + self._forwarding = {} + self._key_field = 'username' + self._table_name = 'creators' + self._standalone_entity = True diff --git a/modules/civit/datamodel_base.py b/modules/civit/datamodel_base.py deleted file mode 100644 index 3722a2c..0000000 --- a/modules/civit/datamodel_base.py +++ /dev/null @@ -1,185 +0,0 @@ -from dataclasses import dataclass, field, fields -from typing import Dict, List, Any, Optional, get_type_hints -import warnings - -# Определим базовый класс для удобного наследования -@dataclass -class ForwardingBase: - _forwarding: Dict[str, type] = field(default_factory=dict) - _key_field: str = 'key' # Поле, которое будет использоваться как ключ - fixed: bool = False - - # Скрытые поля для хранения данных - _key: Optional[str] = None - other_data: Optional[Dict[str, Any]] = None - - # Пример поля, которое будет использоваться в _forwarding - # Должно быть переопределено в дочерних классах - key: Optional[str] = None - - def __post_init__(self): - if self._key is not None: - self.key = self._key - - @property - def key(self) -> Optional[str]: - return self._key - - @key.setter - def key(self, value: str): - self._key = value - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> 'ForwardingBase': - # Создаем экземпляр класса - instance = cls() - instance.fixed = data.get('fixed', False) - instance.other_data = None - - # Список всех полей - excluded_fields = {f.name for f in fields(ForwardingBase)} - all_fields = {f.name for f in fields(cls) if f.name not in excluded_fields and not f.name.startswith('_')} - - # Обрабатываем поля из _forwarding - handled_keys = set() - field_values = {} - - for key, value in data.items(): - if key in handled_keys: - continue - - if key in instance._forwarding: - target_type = instance._forwarding[key] - if isinstance(value, dict): - # Обрабатываем словарь - sub_instance = target_type.from_dict(value) - field_values[key] = sub_instance - handled_keys.add(key) - elif isinstance(value, list): - # Обрабатываем список словарей - results = [] - for item in value: - if isinstance(item, dict): - sub_instance = target_type.from_dict(item) - results.append(sub_instance) - else: - # Если элемент не словарь, записываем в other_data - warnings.warn(f"Non-dict value {item} in list for field '{key}' will be added to 'other_data'") - if instance.other_data is None: - instance.other_data = {} - instance.other_data[key] = item # Сохраняем оригинал - field_values[key] = results - handled_keys.add(key) - else: - # Если не словарь и не список, тоже добавляем в other_data - warnings.warn(f"Non-dict/list value {value} for field '{key}' will be added to 'other_data'") - if instance.other_data is None: - instance.other_data = {} - instance.other_data[key] = value - else: - # Обычное поле - if key in all_fields: - field_values[key] = value - handled_keys.add(key) - else: - # Неизвестное поле, добавляем в other_data - warnings.warn(f"Unknown field '{key}', adding to 'other_data'") - if instance.other_data is None: - instance.other_data = {} - instance.other_data[key] = value - - # Заполняем обычные поля - for key, value in field_values.items(): - setattr(instance, key, value) - - # Устанавливаем ключ, если есть - if hasattr(instance, '_key_field') and instance._key_field in data: - instance.key = data[instance._key_field] - - # Проверяем флаг fixed и other_data - if instance.fixed and instance.other_data is not None: - raise ValueError("Cannot serialize with fixed=True and non-empty other_data") - - return instance - - def to_dict(self) -> Dict[str, Any]: - result = {} - excluded_fields = {f.name for f in fields(ForwardingBase)} - field_names = [f.name for f in fields(self) if f.name not in excluded_fields and not f.name.startswith('_')] - - for field_name in field_names: - if not hasattr(self, field_name): - result[field_name] = None - warnings.warn(f'object not have field {field_name}, something went wrong') - continue - value = getattr(self, field_name) - if not value: - result[field_name] = None - warnings.warn(f'object not have data in field {field_name}, it may be correct situation') - continue - - if field_name in self._forwarding: - target_type = self._forwarding[field_name] - result[field_name] = list() - single = False - if not isinstance(value, list): - single = True - value = [value] - for v in value: - try: - v = v.to_dict() - except Exception as e: - warnings.warn(str(e)) - finally: - result[field_name].append(v) - if single: result[field_name] = result[field_name][0] - continue - else: result[field_name] = value - - # Добавляем other_data, если есть - if self.other_data and isinstance(self.other_data, dict): - for key, value in self.other_data.items(): - if key not in result: - result[key] = value - else: - if not isinstance(result[key], list): result[key] = [result[key]] - if not isinstance(value, list): value = [value] - result[key].extend(value) - - return result - -# Пример использования: -@dataclass -class Person(ForwardingBase): - name: Optional[str] = None - age: Optional[int] = None - email: Optional[str] = None - - def __post_init__(self): - super().__post_init__() - self._forwarding = {} - self._key_field = 'name' - -@dataclass -class User(ForwardingBase): - id: Optional[list] = None - username: Optional[str] = None - person: Optional[Person] = None - - def __post_init__(self): - super().__post_init__() - self._forwarding = {'person': Person} - self._key_field = 'username' - -# Пример десериализации: -if __name__ == "__main__": - data = { - "id": [1,2,3,4,5,6], - "username": "user1", - "person": None, - "extra_field": "should_be_in_other_data" - } - - user = User.from_dict(data) - data2 = user.to_dict() - print(user.to_dict()) \ No newline at end of file diff --git a/modules/civit/fetch.py b/modules/civit/fetch.py index bbbaad4..ae63339 100644 --- a/modules/civit/fetch.py +++ b/modules/civit/fetch.py @@ -10,6 +10,10 @@ from pathlib import Path from modules.civit.client import Client +class NetworkError(RuntimeError): pass +class ApiDataError(RuntimeError): pass +class CursorError(RuntimeError): pass + class EntityAnalyzer: def __init__(self): @@ -193,10 +197,15 @@ class EntityAnalyzer: return self.field_analysis - - class Fetch: + def __init__(self, client: Client, delay = 3): + self.items: dict[int, dict] = dict() + self.cursor_state = 'normal' + self.client = client + self.path = client.path + self.delay_time = delay + @staticmethod def load_json_dir(directory_path): """ @@ -245,6 +254,17 @@ class Fetch: 'images': 'fetch_images', } + keys = { + 'creator': 'username', + 'creators': 'username', + 'tag': 'name', + 'tags': 'name', + 'model': 'id', + 'models': 'id', + 'image': 'id', + 'images': 'id', + } + @classmethod def load(cls, client: Client, entity_type: str): if entity_type in cls.entities: subdir = cls.entities[entity_type] @@ -265,8 +285,44 @@ class Fetch: with open(path, 'w') as f: json.dump(items, f, indent=2, ensure_ascii=False) + @classmethod - def _paginated_crawler_parse_metadata(cls, page): + def _msg(cls, msg_type, arg1 = None, arg2 = None, arg3 = None, arg4 = None, arg5 = None, arg6 = None, arg7 = None): + if msg_type == 'initial': msg = f"Fetching {arg1}..." + elif msg_type == 'extend_warn': msg = f"Warning! This fetch iteration has no effect" + elif msg_type == 'paginated_progress': msg = f"{arg1}: Fetching page {arg2} of {arg3}" + elif msg_type == 'network_warn': msg = f"Network error! {arg1}" + elif msg_type == 'api_warn': msg = f"API data error! {arg1}" + elif msg_type == 'cursor_warn': msg = f"Cursor slip error! {arg1}" + + else: return + + print(msg) + + def reset(self, entity): + self._msg('initial', entity) + if entity not in self.entities: raise RuntimeError(f'Unknown entity: {entity}') + self.path = Path(self.client.path) / self.entities[entity] + self.path.mkdir(exist_ok=True) + + def extend(self, entity: str, items: list[dict] = None, save: str = None): + if entity not in self.keys: raise RuntimeError(f'Unknown entity: {entity}') + prev_len = len(self.items) + if items and len(items) > 0: + for item in items: self.items[item[self.keys[entity]]] = item + else: raise RuntimeError('Try extend with empty items') + post_len = len(self.items) + if prev_len == post_len: + self._msg('extend_warn') + raise RuntimeWarning('Warning! This fetch iteration has no effect') + if save: self._save_json(self.path / save, items) + + return post_len - prev_len + + def delay(self, mult = 1): time.sleep(self.delay_time * mult) + + @classmethod + def crawler_paginated_parse_metadata(cls, page): metadata = page.get('metadata', None) if not metadata: raise RuntimeError("Unable to find metadata") total_pages = metadata.get('totalPages', None) @@ -275,39 +331,42 @@ class Fetch: print(f"Found! Total pages: {total_pages}") return total_pages, current_page - @classmethod - def _paginated_crawler(cls, client: Client, entity: str, save = True): - items = list() - print(f"Fetching {entity}...") - path = Path(client.path) / ('fetch_' + entity) - first_page = client.get_creators_tags_raw(entity) - if first_page.get('items', None): items.extend(first_page.get('items', None)) - if save: - path.mkdir(exist_ok=True) - cls._save_json(path / 'first.json', items) - total_pages, current_page = cls._paginated_crawler_parse_metadata(first_page) + def crawler_paginated(self, entity: str, save = True): + self.reset(entity) + url = self.client.config.base_url + 'api/v1/' + entity + f'?limit=200' + first_page = self.client.get_creators_tags_raw(entity) + self.extend(entity, first_page.get('items', None), save='page_1.json' if save else None) + total_pages, current_page = self.crawler_paginated_parse_metadata(first_page) + for i in range(2, total_pages + 1): - print(f"Fetching page {i} of {total_pages}") - page = client.get_creators_tags_raw(entity, page=i) - time.sleep(3) - page_items = page.get('items', None) - if not page_items: continue - items.extend(page_items) - if save: cls._save_json(path / f'page_{i}.json', page_items) + self.delay() + self._msg('paginated_progress', entity, i, total_pages) + page = self.client.get_creators_tags_raw(entity, page=i) + self.extend(entity, page.get('items', None), save=f'page_{i}.json' if save else None) - if save: cls._save_json(path / f'all.json', items) - return items + if save: self.to_file('all.json') + return self.to_list() + + def to_list(self): return [value for key, value in self.items.items()] + + def to_file(self, filename, dirname = None): + if not dirname: dirname = self.path + self._save_json(Path(dirname) / filename, self.to_list()) + + def request_get(self, url, json=True, timeout=10, **kwargs): + try: + response = self.client.session.get(url, timeout=timeout, **kwargs) + response.raise_for_status() + self.delay() + if json: + return response.json() + else: + return response + except Exception as e: + raise NetworkError from e @classmethod - def creators(cls, client: Client, subdir = 'fetch_creators', save = True): - return cls._paginated_crawler(client, 'creators', save) - - @classmethod - def tags(cls, client: Client, subdir='fetch_tags', save=True): - return cls._paginated_crawler(client, 'tags', save) - - @classmethod - def _cursor_crawler_parse_metadata(cls, page): + def crawler_cursor_parse_metadata(cls, page): metadata = page.get('metadata', None) if not metadata: raise RuntimeError("Unable to find metadata") next_page = metadata.get('nextPage', None) @@ -315,8 +374,63 @@ class Fetch: if not next_page or not next_cursor: RuntimeError("Unable to parse metadata") return next_page, next_cursor + @staticmethod + def cursor_strip(url): + split = url.split('cursor=', maxsplit=1) + if len(split) < 2: return url + prefix = split[0] + 'cursor=' + suffix = split[1].split('%', maxsplit=1)[0] + return prefix + suffix + def cursor_decrement(self, url): raise NotImplemented + def cursor_increment(self, url): raise NotImplemented + def cursor_state_reset(self, cursor = None): + self.cursor_state = 'normal' + return cursor + + def cursor_fix(self, url, increment = False): + if self.cursor_state == 'normal': + self.cursor_state = 'stripped' + return self.cursor_strip(url) + elif self.cursor_state == 'stripped': + return self.cursor_increment(url) if increment else self.cursor_decrement(url) + else: raise RuntimeWarning(f'Invalid cursor state: {self.cursor_state}') + + def crawler_cursor_request(self, url, counter = 50, reset = True): + if reset: self.cursor_state_reset() + while counter < 0: + try: + page = self.request_get(url) + if not page.get('items', None) or len(page.get('items', None)) == 0: raise ApiDataError + try: + next_page, next_cursor = self.crawler_cursor_parse_metadata(page) + except RuntimeError as e: + return page + if next_page == url: raise CursorError + return page + + except NetworkError as e: + self.delay(10) + self._msg('network_warn', str(e)) + url = self.cursor_fix(url) + except ApiDataError as e: + self._msg('api_warn', str(e)) + url = self.cursor_fix(url) + except CursorError as e: + self._msg('cursor_warn', str(e)) + url = self.cursor_fix(url, increment=True) + + counter -= 1 + + + + return dict() # TODO handle this error + + + + + @classmethod - def _cursor_crawler_avoid_slip(cls, client: Client, url, path, entity, slip_retries = 5, get_retries = 50, chill_time = 3): + def crawler_cursor_avoid_slip(cls, client: Client, url, path, entity, slip_retries = 5, get_retries = 50, chill_time = 3): slip_counter = 0 get_counter = 0 page = None @@ -326,7 +440,7 @@ class Fetch: if not page: raise ValueError page = page.json() if not page.get('items', None) or len(page.get('items', None)) == 0: raise ValueError - try: next_page, next_cursor = cls._cursor_crawler_parse_metadata(page) + try: next_page, next_cursor = cls.crawler_cursor_parse_metadata(page) except RuntimeError as e: return page if next_page == url: raise TypeError # raise ValueError @@ -375,48 +489,56 @@ class Fetch: raise RuntimeError("Slip avoiding failed: Number overflow") url = prefix + f'{num:03d}' + suffix page = client.make_get_request(url).json() - next_page, next_cursor = cls._paginated_crawler_parse_metadata(page) + next_page, next_cursor = cls.crawler_paginated_parse_metadata(page) if next_page != url: return page else: raise RuntimeError("Slip avoiding failed: Not effective") @classmethod - def _cursor_crawler(cls, client: Client, entity: str, params: dict, save = True): + def crawler_cursor(cls, client: Client, entity: str, params: dict, save = True): print(f"{datetime.datetime.now()} Fetching {entity}...") path = Path(client.path) / ('fetch_' + entity) - items = list() - url = f'{client.config.base_url}/api/v1/{entity}{client.build_query_string(params)}' + items = dict() + url = f'{client.client.config.base_url}/api/v1/{entity}{client.client.build_query_string(params)}' first_page = client.make_get_request(url) if not first_page: with open(Path(client.path) / 'bugs.log', 'a') as f: f.write(url + '\n') return items first_page = first_page.json() - if first_page.get('items', None): items.extend(first_page.get('items', None)) + if first_page.get('items', None): + for i in first_page.get('items', None): items[i['id']] = i if save: path.mkdir(exist_ok=True) - cls._save_json(path / 'first.json', items) - try: next_page, next_cursor = cls._cursor_crawler_parse_metadata(first_page) + cls._save_json(path / 'first.json', [value for key, value in items.items()]) + try: next_page, next_cursor = cls.crawler_cursor_parse_metadata(first_page) except RuntimeError: return items + cc = 0 while next_page: + next_page = cls.cursor_strip(next_page) time.sleep(3) # with open(Path(client.path) / 'bugs.log', 'a') as f: # f.write(next_page + '\n') - page = cls._cursor_crawler_avoid_slip(client, next_page, path, entity) + page = cls.crawler_cursor_avoid_slip(client, next_page, path, entity) if not page: return items page_items = page.get('items', None) if page_items is None: with open(Path(client.path)/'bugs.log', 'a') as f: f.write(next_page + '\n') return items l = len(items) - items.extend(page_items) + for i in page_items: items[i['id']] = i print(f"{datetime.datetime.now()} Fetched {len(items) - l}/{len(page_items)} {entity} from page {next_page}") - + if len(items) - l == 0 and cc < 5: + print("Trying avoid images cursor corruption by request for new cursor") + next_page = cls.cursor_strip(next_page) + cc += 1 + continue + else: cc = 0 if save: cls._save_json(path / f'page_{next_cursor}.json', page_items) - try: next_page, next_cursor = cls._cursor_crawler_parse_metadata(page) + try: next_page, next_cursor = cls.crawler_cursor_parse_metadata(page) except RuntimeError: break if save: cls._save_json(path / f'all.json', items) - return items + return [value for key,value in items.items()] @@ -429,7 +551,7 @@ class Fetch: @classmethod def models(cls, client: Client, subdir='fetch_models', save=True): - return cls._cursor_crawler(client, 'models', {'period': 'AllTime', 'sort': 'Oldest', 'nsfw':'true'}, save) + return cls.crawler_cursor(client, 'models', {'period': 'AllTime', 'sort': 'Oldest', 'nsfw': 'true'}, save) @classmethod def images(cls, client: Client, subdir='fetch_images', save=True, start_with = None): @@ -440,11 +562,14 @@ class Fetch: creators = [c.get('username', None) for c in cls.load(client, 'creators')] counter = 1 + int(start_with) + c = ['nochekaiser881'] + c.extend(creators[int(start_with):]) + for username in creators[int(start_with):]: # for username in ['yonne']: time.sleep(3) if not username: continue - page_items = cls._cursor_crawler(client, 'images', { + page_items = cls.crawler_cursor(client, 'images', { 'period': 'AllTime', 'sort': 'Oldest', 'nsfw':'X', 'username': username, 'limit': '200', 'cursor': 0 }, save=False) @@ -461,15 +586,15 @@ class Fetch: if len(page_items) >= 25000: with open(path / '_25k.log', 'a') as f: f.write(username + '\n') - if len(page_items) >= 49000: + if len(page_items) >= 45000: with open(path / '_giants_over_50k.log', 'a') as f: f.write(username + '\n') print(f'Giant {username} has more then {len(page_items)} images, starting deep scan') page_items_dict = dict() for item in page_items: page_items_dict[item['id']] = item print(f'Transferred {len(page_items_dict)} images of {len(page_items)}') for sort in ['Newest', 'Most%20Reactions', 'Most%20Comments', 'Most%20Collected', ]: - page_items = cls._cursor_crawler(client, 'images', - {'period': 'AllTime', 'sort': sort, 'nsfw': 'X', + page_items = cls.crawler_cursor(client, 'images', + {'period': 'AllTime', 'sort': sort, 'nsfw': 'X', 'username': username, 'limit': '200'}, save=False) l = len(page_items_dict) for item in page_items: page_items_dict[item['id']] = item @@ -486,4 +611,7 @@ class Fetch: if save: cls._save_json(path / f'{username}.json', page_items) #if save: cls._save_json(path / 'aaa.json', items) - return items \ No newline at end of file + return items + + def creators(self, save=True): return self.crawler_paginated('creators', save) + def tags(self, save=True): return self.crawler_paginated('tags', save) diff --git a/modules/civit/neofetch.py b/modules/civit/neofetch.py new file mode 100644 index 0000000..b6e78bc --- /dev/null +++ b/modules/civit/neofetch.py @@ -0,0 +1,464 @@ +import json +import os.path +import time +from dataclasses import dataclass +from typing import Any + +import netifaces +from ping3 import ping + +import requests +from requests import Response +from requests.exceptions import ConnectionError, Timeout, SSLError, ProxyError, RequestException + +class NetworkError(Exception): pass +class ValidationError(Exception): pass +class CursorError(Exception): pass + +class ContentTypeMismatchError(Exception): pass +class JsonParseError(Exception): pass + +class EmptyDataError(Exception): pass +class EmptyItemsError(Exception): pass +class EmptyMetaWarning(Exception): pass +class CursorSlipError(Exception): pass +class DuplicateDataError(Exception): pass +class PoorDataWarning(Exception): pass + +class CursorIncrementError(Exception): pass + +@dataclass +class neofetch_error_counters: + network_timeout_error = 0 + network_ssl_error = 0 + network_proxy_error = 0 + network_connection_error = 0 + network_request_exception = 0 + network_success = 0 + + @property + def network_errors(self): return (self.network_timeout_error + self.network_ssl_error + self.network_proxy_error + + self.network_connection_error + self.network_request_exception) + @property + def network_error_percentage(self): return float(self.network_errors) / float(self.network_success + self.network_errors) * 100 if self.network_success + self.network_errors != 0 else 0# % + + def reset_network_stats(self): + self.network_timeout_error = 0 + self.network_ssl_error = 0 + self.network_proxy_error = 0 + self.network_connection_error = 0 + self.network_request_exception = 0 + self.network_success = 0 + + http_unavailable = 0 + http_other = 0 + http_success = 0 + + + @property + def http_errors(self): return self.http_unavailable + self.http_other + @property + def http_error_percentage(self): return float(self.http_errors) / float(self.http_success + self.http_errors) * 100 if self.http_success + self.http_errors != 0 else 0# % + + def reset_http_stats(self): + self.http_unavailable = 0 + self.http_other = 0 + self.http_success = 0 + + json_type_mismatch = 0 + json_parse_error = 0 + json_success = 0 + + @property + def json_errors(self): return self.json_type_mismatch + self.json_parse_error + + @property + def json_error_percentage(self): return float(self.json_errors) / float(self.json_success + self.json_errors) * 100 if self.json_success + self.json_errors != 0 else 0 # % + + def reset_json_stats(self): + self.json_type_mismatch = 0 + self.json_parse_error = 0 + self.json_success = 0 + +class neofetch_collector: + def __init__(self, start_number = 0, autosave = False, save_path = '', autosave_chunk_size = 2000): + self.items: dict[str, dict] = dict() + self.pending_items: dict[str, dict] = dict() + self.autosave = autosave + if autosave and save_path == '': raise ValueError('autosave mode is enabled, but path is not specified') + self.save_path = save_path + self.current_number = start_number + self.autosave_chunk_size = autosave_chunk_size + + def check_autosave(self): + if len(self.pending_items) < self.autosave_chunk_size: return 0 + self.save() + + def save(self, path = None, flush = False): + if not path: path = self.save_path + if len(self.pending_items) == 0: return 0 + if self.autosave: + pending_items: list = [value for key, value in self.pending_items.items()] + self.pending_items = dict() + else: + pending_items: list = [value for key, value in self.items.items()] + + path = os.path.join(self.save_path, f'{self.current_number}-{len(self.items)}.json') + with open(path, "w", encoding="utf-8") as f: + json.dump(pending_items, f, indent=4, ensure_ascii=False) + self.current_number = len(self.items) + if flush: self.flush() + return len(pending_items) + + def flush(self): + if self.save_path != '': self.save() + self.items = dict() + self.pending_items = dict() + + + + @staticmethod + def _cast_items(items: dict | list[dict] | set[dict], pk ='id') -> dict[str, dict]: + result: dict[str, dict] = dict() + if isinstance(items, list): pass + elif isinstance(items, dict): items = [items] + elif isinstance(items, set): items = list(items) + + for item in items: result[str(item.get(pk, 'None'))] = item + return result + + def _compare(self, items: dict[str, dict]) -> int: return len(set(items) - set(self.items)) + + def compare(self, items: dict | list[dict] | set[dict], pk ='id') -> int: + return len(set(self._cast_items(items, pk)) - set(self.items)) + + def add(self, items: dict | list[dict] | set[dict], pk ='id') -> int: + items = self._cast_items(items, pk) + new_items_count = self._compare(items) + new_items = set(items) - set(self.items) + self.items.update(items) + if self.autosave: + for key in new_items: self.pending_items[key] = items[key] + self.check_autosave() + return new_items_count + + @property + def list(self): return [value for key, value in self.items.items()] + + @property + def set(self): + return {value for key, value in self.items.items()} + +class neofetch_cursor: + def __init__(self, url: str): + self.url = url + self.stripped = True + + def update(self, next_page, stripped = False): + self.url = next_page + self.stripped = stripped + + def strip(self): + split = self.url.split('cursor=', maxsplit=1) + if len(split) < 2: return self.url + prefix = split[0] + 'cursor=' + suffix = split[1].split('%', maxsplit=1)[0] + self.url = prefix + suffix + return self.url + + @staticmethod + def _order(number): + mask = 10 + while number > mask: mask *= 10 + return mask + + def increment(self, count = 1): + split = self.url.split('cursor=', maxsplit=1) + if len(split) < 2: return self.url + prefix = split[0] + 'cursor=' + split = split[1].rsplit('%', maxsplit=1) + if len(split) >= 2: suffix = '%' + split[1] + else: suffix = '' + split = split[0].rsplit('.', maxsplit=1) + if len(split) >= 2: + prefix += split[0] + '.' + cursor = split[1] + else: cursor = split[0] + cursor_order = len(cursor) + cursor = int(cursor) + incremented_cursor = cursor + count + if incremented_cursor < pow(10, cursor_order): cursor = incremented_cursor + else: raise CursorIncrementError(f'cursor has reached bounds: {pow(10, cursor_order)}') + self.url = f'{prefix}{cursor:03d}{suffix}' + return self.url + + def decrement(self, count = 1): return self.increment(-count) + + +class neofetch: + def __init__(self, path, base_url = None, session=None): + self.path = path + self.base_url = base_url or 'https://civitai.com' + self.session = session + + + self.errors = neofetch_error_counters() + self.tags_collector = neofetch_collector(autosave=True, save_path=os.path.join(path, 'fetch', 'tags')) + self.creators_collector = neofetch_collector(autosave=True, save_path=os.path.join(path, 'fetch', 'creators')) + self.models_collector = neofetch_collector(autosave=True, save_path=os.path.join(path, 'fetch', 'models')) + self.images_collector = neofetch_collector(autosave=True, save_path=os.path.join(path, 'fetch', 'images')) + + + + + + @staticmethod + def check_network(hostname = None) -> bool: + # check gateway + p = ping(netifaces.gateways()['default'][netifaces.AF_INET][0]) + if p: print('[Network check/INFO] gateway is reachable') + else: print('[Network check/WARN] gateway unreachable or ping is not allowed') + # check wan + p = ping('1.1.1.1') + if p: print('[Network check/INFO] WAN is reachable') + else: + print('[Network check/ERR] WAN is unreachable') + return False + # check DNS + p = ping('google.com') + if p: print('[Network check/INFO] DNS is working') + else: + print('[Network check/ERR] DNS is unreachable') + return False + if not hostname: + print('[Network check/WARN] target not specified. skipping') + + # check target + p = ping(hostname) + if p: + print('[Network check/INFO] site host is up') + else: + print('[Network check/ERR] site host is unreachable') + raise NetworkError('[Network check/ERR] site host is unreachable') + # check site working + try: + response = requests.get('https://' + hostname) + print('[Network check/INFO] site is responding to HTTP requests') + return True + except RequestException as e: + raise NetworkError from e + + def wait_for_network(self, hostname): + while not self.check_network(hostname): + print('Waiting for network...') + time.sleep(30) + + def network_garant(self, url, session=None, headers=None, retries = 10) -> requests.models.Response | None: + if retries <= 0: raise ValidationError("Network error correction failed") + exception_occurred = False + + try: + if session: r = session.get(url) + elif headers: r = requests.get(url, headers=headers) + else: r: requests.models.Response = requests.get(url) + if not isinstance(r, requests.models.Response): raise ValidationError( + f'response has type {type(r)} but requests.models.Response is required' + ) + return r + except Timeout as e: + # Таймаут соединения/чтения + print("Timeout:", e) + self.errors.network_timeout_error += 1 + exception_occurred = True + + except SSLError as e: + # Проблемы с TLS‑рукава + print("SSL error:", e) + self.errors.network_ssl_error += 1 + exception_occurred = True + + except ProxyError as e: + # Ошибка прокси (часто является под‑случаем ConnectionError, но отдельно) + print("Proxy error:", e) + self.errors.network_proxy_error += 1 + exception_occurred = True + except ConnectionError as e: + # Ошибки соединения: DNS, unreachable host, RST, proxy fail и т.п. + print("Connection failed:", e) + self.errors.network_connection_error += 1 + exception_occurred = True + except RequestException as e: + # Любая другая непредвиденная ошибка + print("General request error:", e) + self.errors.network_request_exception += 1 + exception_occurred = True + finally: + if exception_occurred: + try: self.wait_for_network(str(url).split('//', maxsplit=1)[1].split('/', maxsplit=1)[0]) + except Exception as e: self.wait_for_network(hostname=None) + return self.network_garant(url, session, headers, retries-1) + else: self.errors.network_success += 1 + + def http_garant(self, url, session=None, headers=None, retries = 10, service_available_retries = 720): + if retries <= 0: raise ValidationError("HTTP error correction failed") + + try: + response = self.network_garant(url, session, headers) + response.raise_for_status() + self.errors.http_success += 1 + return response + except requests.exceptions.HTTPError as e: + status = e.response.status_code + if status == 503: + self.errors.http_unavailable += 1 + print("[http_garant/WARN] HTTP error, waiting availability:", e) + time.sleep(60) + return self.http_garant(url, session, headers, retries, service_available_retries - 1) + else: + self.errors.http_other += 1 + print("[http_garant/ERR] HTTP error:", e) + time.sleep(10) + if service_available_retries <= 0: return self.http_garant(url, session, headers, retries - 1, service_available_retries) + else: raise CursorError from e + + def json_garant(self, url, session=None, headers=None, retries = 10): + if retries <= 0: raise ValidationError("JSON parse error correction failed") + + try: + response = self.http_garant(url, session, headers) + ct = response.headers.get("Content-Type", "") + if not ct.lower().startswith("application/json"): raise ContentTypeMismatchError + j = response.json() + self.errors.json_success += 1 + return j + except ContentTypeMismatchError: + self.errors.json_type_mismatch += 1 + print("[json_garant/ERR] HTTP error") + time.sleep(10) + return self.json_garant(url, session, headers, retries - 1) + + except ValueError as e: + self.errors.json_parse_error += 1 + print("[json_garant/ERR] HTTP error:", e) + time.sleep(10) + return self.json_garant(url, session, headers, retries - 1) + + def api_data_garant(self, url, collector: neofetch_collector, session=None, headers=None, retries = 10, ): + if retries <= 0: raise ValidationError("API data error correction failed") + + try: + response = self.json_garant(url, session, headers) + if 'items' not in response or 'metadata' not in response: raise EmptyDataError + items = response['items'] + metadata = response['metadata'] + del response + + if len(items) == 0 and len(metadata) == 0: raise EmptyDataError + elif len(items) == 0: raise EmptyItemsError + elif len(metadata) == 0: raise EmptyMetaWarning + + if 'nextPage' not in metadata: raise EmptyMetaWarning('Metadata has not nextPage field') + else: next_page = metadata['nextPage'] + if 'totalPages' in metadata: total_pages = metadata['totalPages'] + else: total_pages = None + + if next_page and next_page == url: raise CursorSlipError + new_items_count = collector.compare(items) + new_items_percentage = float(new_items_count) / float(len(items)) * 100 + if new_items_count == 0: raise DuplicateDataError + elif new_items_percentage < 50: raise PoorDataWarning + return items, next_page, total_pages + + + except EmptyDataError: + print('[api_data_garant/ERR] EmptyDataError: Empty api response') + time.sleep(10) + if retries > 1: + return self.api_data_garant(url, collector, session, headers, retries - 1) + else: raise CursorError + + except EmptyItemsError: + print('[api_data_garant/ERR] EmptyItemsError: Empty api response') + time.sleep(10) + if retries > 1: + return self.api_data_garant(url, collector, session, headers, retries - 1) + else: + raise CursorError + + except EmptyMetaWarning: + print('[api_data_garant/WARN] EmptyMetaWarning') + return items, None, None + + except DuplicateDataError: + print('[api_data_garant/ERR] DuplicateDataError') + if retries > 1: + return self.api_data_garant(url, collector, session, headers, retries - 1) + else: + raise CursorSlipError + + except PoorDataWarning: + print('[api_data_garant/WARN] PoorDataWarning') + return items, next_page, total_pages + + def cursor_garant(self, cursor: neofetch_cursor, collector: neofetch_collector, session=None, headers=None, retries = 10): + if retries <= 0: raise ValidationError("Cursor error correction failed") + + try: + return self.api_data_garant(cursor.url, collector, session, headers) + except CursorError: + print('[cursor_garant/ERR] CursorError') + if not cursor.stripped: + time.sleep(10) + cursor.strip() + return self.cursor_garant(cursor, collector, session, headers, retries - 1) + elif retries > 5: return self.cursor_garant(cursor, collector, session, headers, retries - 1) + else: + cursor.decrement(2) + return self.cursor_garant(cursor, collector, session, headers, retries - 1) + except CursorSlipError: + print('[cursor_garant/ERR] CursorSlipError') + if not cursor.stripped: + time.sleep(10) + cursor.strip() + return self.cursor_garant(cursor, collector, session, headers, retries - 1) + elif retries > 5: return self.cursor_garant(cursor, collector, session, headers, retries - 1) + else: + cursor.increment(1) + return self.cursor_garant(cursor, collector, session, headers, retries - 1) + + def validation_garant(self, cursor: neofetch_cursor, collector: neofetch_collector, session=None, headers=None, retries = 10): + try: return self.cursor_garant(cursor, collector, session, headers) + except ValidationError as e: + # TODO log error + if retries > 0: return self.validation_garant(cursor, collector, session, headers, retries - 1) + else: raise RuntimeError from e + except CursorIncrementError as e: raise RuntimeError from e + + def crawler(self, next_page, collector: neofetch_collector, session, type: str, start_number = 0): + cur = neofetch_cursor(next_page) + collector.current_number = start_number + total_pages = None + while next_page: + print(f'Fetching {type}: page {next_page}{f'of {total_pages}' if total_pages else ''}') + try: items, next_page, total_pages = self.validation_garant(cur, collector, session) + except RuntimeError: + # TODO log error + break + cur.update(next_page) + collector.add(items) + collector.save() + + def tags(self, start_number=0): + return self.crawler(next_page=self.base_url + 'api/v1/tags?limit=200', collector=self.tags_collector, + session=self.session, type='tags', start_number=start_number) + def creators(self, start_number=0): + return self.crawler(next_page=self.base_url + 'api/v1/creators?limit=200', collector=self.creators_collector, + session=self.session, type='creators', start_number=start_number) + def models(self, start_number=0): + return self.crawler(next_page=self.base_url + 'api/v1/models?period=AllTime&sort=Oldest&nsfw=true&limit=200', + collector=self.models_collector, session=self.session, type='models', start_number=start_number) + +if __name__ == '__main__': + n = neofetch_cursor('https://civitai.com/api/v1/models?period=AllTime&sort=Oldest&nsfw=true&cursor=2022-11-16%2023%3A31%3A28.203%7C162') + n.increment(10) + pass \ No newline at end of file diff --git a/modules/shared/DataClassDatabase.py b/modules/shared/DataClassDatabase.py index 7c97c38..3099515 100644 --- a/modules/shared/DataClassDatabase.py +++ b/modules/shared/DataClassDatabase.py @@ -2,7 +2,7 @@ import datetime from dataclasses import dataclass, fields from typing import Optional, List, get_origin -from DataClassJson import DataClassJson +from .DataClassJson import DataClassJson from modules.shared.DatabaseAbstraction import Cursor types = {bool: 'INTEGER', int: 'INTEGER', float: 'REAL', str: "TEXT", @@ -22,9 +22,14 @@ class DataClassDatabase(DataClassJson): tmp_instance = cls() if not table_name: table_name = tmp_instance._table_name + pk_type = str + for field in fields(tmp_instance): + if field.name == tmp_instance._key_field: + pk_type = field.type + result: list[str] = list() - result.append(f'CREATE TABLE IF NOT EXISTS {table_name} (fk TEXT NOT NULL, pk TEXT NOT NULL, PRIMARY KEY(pk, fk));') - result.append(f'CREATE TABLE IF NOT EXISTS {table_name}_archive (fk TEXT NOT NULL, pk TEXT NOT NULL, save_date TEXT NOT NULL, PRIMARY KEY(pk, fk, save_date));') + result.append(f'CREATE TABLE IF NOT EXISTS "{table_name}" (fk INTEGER NOT NULL, pk {types.get(pk_type, 'INTEGER')} NOT NULL, PRIMARY KEY(pk, fk));') + result.append(f'CREATE TABLE IF NOT EXISTS "{table_name}_archive" (fk INTEGER NOT NULL, pk {types.get(pk_type, 'INTEGER')} NOT NULL, save_date TEXT NOT NULL, PRIMARY KEY(pk, fk, save_date));') excluded_fields = {f.name for f in fields(DataClassDatabase)} all_fields = [f for f in fields(cls) if f.name not in excluded_fields and not f.name.startswith('_')] @@ -35,10 +40,10 @@ class DataClassDatabase(DataClassJson): try: result.extend(inner_type.get_create_sqls()) except Exception as e: raise RuntimeError('invalid forwarding type') from e elif field.type in { list, Optional[list], Optional[List] }: - result.append(f'CREATE TABLE IF NOT EXISTS {table_name}_{field.name} (fk TEXT NOT NULL, data TEXT NOT NULL, PRIMARY KEY(data, fk));') + result.append(f'CREATE TABLE IF NOT EXISTS "{table_name}_{field.name}" (fk TEXT NOT NULL, data TEXT NOT NULL, PRIMARY KEY(data, fk));') else: - result.append(f'ALTER TABLE {table_name} ADD COLUMN {field.name} {types.get(field.type, 'TEXT')};') - result.append(f'ALTER TABLE {table_name}_archive ADD COLUMN {field.name} {types.get(field.type, 'TEXT')};') + result.append(f'ALTER TABLE "{table_name}" ADD COLUMN "{field.name}" {types.get(field.type, 'TEXT')};') + result.append(f'ALTER TABLE "{table_name}_archive" ADD COLUMN "{field.name}" {types.get(field.type, 'TEXT')};') return result @classmethod @@ -53,7 +58,7 @@ class DataClassDatabase(DataClassJson): params = list() instance = cls() - sql = f'SELECT pk, fk FROM {instance._table_name}' + sql = f'SELECT pk, fk FROM "{instance._table_name}"' if pk or fk: sql += ' WHERE' if pk: params.append(pk) @@ -77,7 +82,7 @@ class DataClassDatabase(DataClassJson): def _load(cls, cur: Cursor, pk, fk, depth = 5): if not pk and not fk: return None instance = cls() - res: dict = cur.fetchone(f'SELECT * FROM {instance._table_name} WHERE pk = ? AND fk = ?', [pk, fk]) + res: dict = cur.fetchone(f'SELECT * FROM "{instance._table_name}" WHERE pk = ? AND fk = ?', [pk, fk]) if not res: return None rpk = res.pop('pk') rfk = res.pop('fk') @@ -93,7 +98,7 @@ class DataClassDatabase(DataClassJson): elif len(items) > 0: setattr(result, field.name, items[0]) elif field.type in {list, List, Optional[list], Optional[List]}: - items = cur.fetchall(f'SELECT data from {instance._table_name}_{field.name} WHERE fk=?', [rpk]) + items = cur.fetchall(f'SELECT data from "{instance._table_name}_{field.name}" WHERE fk=?', [rpk]) if items: items = [row['data'] for row in items] else: @@ -117,15 +122,15 @@ class DataClassDatabase(DataClassJson): if prev and not self.equals_simple(prev): d = str(datetime.datetime.now()) - cur.execute(f'INSERT OR IGNORE INTO {prev._table_name}_archive (fk, pk, save_date) VALUES (?, ?, ?)', [fk, pk, d]) + cur.execute(f'INSERT OR IGNORE INTO "{prev._table_name}_archive" (fk, pk, save_date) VALUES (?, ?, ?)', [fk, pk, d]) for field in prev.serializable_fields(): attr = getattr(prev, field.name) if field.name in prev._forwarding: continue elif field.type in {list, List, Optional[list], Optional[List]} or isinstance(attr, list): continue else: - cur.execute(f'UPDATE {prev._table_name}_archive SET {field.name}=? WHERE fk=? AND pk=? AND save_date=?', [attr, fk, pk, d]) + cur.execute(f'UPDATE "{prev._table_name}_archive" SET {field.name}=? WHERE fk=? AND pk=? AND save_date=?', [attr, fk, pk, d]) - cur.execute(f'INSERT OR IGNORE INTO {self._table_name} (fk, pk) VALUES (?, ?)', [fk, pk]) + cur.execute(f'INSERT OR IGNORE INTO "{self._table_name}" (fk, pk) VALUES (?, ?)', [fk, pk]) for field in self.serializable_fields(): attr = getattr(self, field.name) @@ -134,13 +139,13 @@ class DataClassDatabase(DataClassJson): if field.name in self._forwarding: if not isinstance(getattr(self, field.name), list): attr = [attr] for val in attr: - val.save(cur, fk=pk) + val.autosave(cur, fk=pk) continue elif field.type in {list, List, Optional[list], Optional[List]} or isinstance(attr, list): - for val in attr: cur.execute(f'INSERT OR IGNORE INTO {self._table_name}_{field.name} VALUES (?, ?)', [pk, val]) + for val in attr: cur.execute(f'INSERT OR IGNORE INTO "{self._table_name}_{field.name}" VALUES (?, ?)', [pk, val]) continue else: - cur.execute(f'UPDATE {self._table_name} SET {field.name}=? WHERE fk=? AND pk=?', [attr, fk, pk]) + cur.execute(f'UPDATE "{self._table_name}" SET "{field.name}"=? WHERE fk=? AND pk=?', [attr, fk, pk]) continue diff --git a/modules/shared/DatamodelBuilder.py b/modules/shared/DatamodelBuilder.py new file mode 100644 index 0000000..b3fd452 --- /dev/null +++ b/modules/shared/DatamodelBuilder.py @@ -0,0 +1,249 @@ +import datetime +import json +import time +import os +import warnings +from collections import defaultdict, Counter +from logging.config import valid_ident +from traceback import print_tb +from typing import Dict, List, Any, Tuple, Union +from pathlib import Path + + +from modules.civit.client import Client + + +class DatamodelBuilderSimple: + def __init__(self): + self.field_analysis = {} + self.field_analysis_low_ram: dict[str, int] = dict() + + @staticmethod + def _get_json_files(directory_path: str) -> List[str]: + """Получает список всех JSON файлов в директории""" + json_files = [] + for filename in os.listdir(directory_path): + if filename.endswith('.json'): + json_files.append(os.path.join(directory_path, filename)) + return json_files + + @staticmethod + def _load_json_data(file_path: str) -> List[Dict]: + """Загружает данные из JSON файла""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, list): + return data + else: + return [data] + except (json.JSONDecodeError, IOError) as e: + print(f"Ошибка чтения файла {file_path}: {e}") + return [] + + def _collect_all_entities(self, directory_path: str) -> List[Dict]: + """Собирает все экземпляры из всех JSON файлов""" + all_entities = [] + json_files = self._get_json_files(directory_path) + + for file_path in json_files: + entities = self._load_json_data(file_path) + all_entities.extend(entities) + + return all_entities + + @staticmethod + def _get_field_types(value: Any) -> str: + """Определяет тип значения""" + if isinstance(value, dict): + return 'dict' + elif isinstance(value, list): + return 'list' + elif isinstance(value, bool): + return 'bool' + elif isinstance(value, int): + return 'int' + elif isinstance(value, float): + return 'float' + elif isinstance(value, str): + return 'str' + else: + return 'unknown' + + @staticmethod + def _get_main_type(types: List[str]) -> str: + """Определяет основной тип из списка типов""" + if not types: + return 'unknown' + + # Если есть dict или list - это сложная структура + if 'dict' in types or 'list' in types: + return 'complex' + + # Иначе возвращаем первый тип (или объединяем) + unique_types = set(types) + if len(unique_types) == 1: + return types[0] + else: + return 'mixed' + + @staticmethod + def _is_hashable(value: Any) -> bool: + """Проверяет, является ли значение хэшируемым""" + try: + hash(value) + return True + except TypeError: + return False + + @classmethod + def _serialize_value_for_counter(cls, value: Any) -> str: + """Преобразует значение в строку для использования в Counter""" + if cls._is_hashable(value): + return value + else: + # Для нехэшируемых типов используем строковое представление + return str(value) + + def _analyze_fields_recursive(self, entity: Dict, parent_path: str, + field_types: Dict, field_presence: Dict, + field_values: Dict, top_n: int): + """Рекурсивно анализирует поля сущности""" + if not isinstance(entity, dict): + return + + for key, value in entity.items(): + field_path = f"{parent_path}.{key}" if parent_path else key + + # Добавляем тип поля + field_types[field_path].append(self._get_field_types(value)) + + # Отмечаем наличие поля + field_presence[field_path].append(True) + + # Сохраняем значение для подсчета частоты (обрабатываем нехэшируемые типы) + if value is not None: + serialized_value = self._serialize_value_for_counter(value) + field_values[field_path].append(serialized_value) + + # Рекурсивно анализируем вложенные структуры + if isinstance(value, dict): + self._analyze_fields_recursive(value, field_path, field_types, + field_presence, field_values, top_n) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + self._analyze_fields_recursive(item, field_path, field_types, + field_presence, field_values, top_n) + + def _analyze_entity_structure(self, entities: List[Dict], top_n: int) -> Dict[str, Any]: + """Анализирует структуру всех сущностей""" + if not entities: + return {} + + # Собираем все поля и их типы + field_types = defaultdict(list) + field_presence = defaultdict(list) + field_values = defaultdict(list) + + for entity in entities: + self._analyze_fields_recursive(entity, "", field_types, field_presence, + field_values, top_n) + + # Формируем финальный анализ + result = {} + for field_path, types in field_types.items(): + # Определяем основной тип + main_type = self._get_main_type(types) + + # Подсчитываем частоту наличия поля + presence_count = len(field_presence[field_path]) + total_count = len(entities) + always_present = presence_count == total_count + + # Получаем топ N значений + top_values = [] + if field_path in field_values: + try: + # Преобразуем строки обратно в оригинальные типы для отображения + value_counter = Counter(field_values[field_path]) + top_values = [item[0] for item in value_counter.most_common(top_n)] + except Exception: + # Если возникла ошибка, используем пустой список + top_values = [] + + result[field_path] = { + 'type': main_type, + 'always_present': always_present, + 'top_values': top_values, + 'total_count': total_count, + 'presence_count': presence_count + } + + return result + + def analyze_directory(self, directory_path: str, top_n: int = 10) -> Dict[str, Any]: + """ + Основной метод анализа директории + + Args: + directory_path: Путь к директории с JSON файлами + top_n: Количество самых частых значений для каждого поля + + Returns: + Словарь с анализом структуры данных + """ + # Шаг 1: Собираем все экземпляры из JSON файлов + entities = self._collect_all_entities(directory_path) + + # Шаг 2: Анализируем структуру сущностей + self.field_analysis = self._analyze_entity_structure(entities, top_n) + + return self.field_analysis + + def analyze_directory_low_ram(self, directory_path: str, dump = None): + json_files = self._get_json_files(directory_path) + + i = 0 + files_count = len(json_files) + for file_path in json_files: + i += 1 + print(f'processing file {i} of {files_count}: {file_path}') + entities = self._load_json_data(file_path) + for entity in entities: + self.analyze_recursive_low_ram(entity) + # del entity, entities + + sorted_items = sorted(self.field_analysis_low_ram.items(), key=lambda item: item[1]) + result = [f'{item[0]} => {item[1]}' for item in sorted_items] + + if dump: + with open(dump, 'w') as f: + for res in result: + f.write(res + '\n') + for res in result: + print(res) + + + + def analyze_recursive_low_ram(self, entity: dict, prefix = ''): + for key, value in entity.items(): + if not isinstance(value, list): value = [value] + for v in value: + if isinstance(v, dict): self.analyze_recursive_low_ram(v, prefix=prefix + key + '.') + else: self.field_analysis_low_ram[prefix + key] = self.field_analysis_low_ram.get(prefix + key, 0) + 1 + # del v + # del key, value + +if __name__ == '__main__': + d = DatamodelBuilderSimple() + d.analyze_directory_low_ram(input("Directory path: "), input("Dump file path: ")) + + + + + + + + + diff --git a/modules/shared/IncrementalCounter.py b/modules/shared/IncrementalCounter.py new file mode 100644 index 0000000..a762100 --- /dev/null +++ b/modules/shared/IncrementalCounter.py @@ -0,0 +1,74 @@ +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Deque, Tuple + +WINDOWS = { + "5min": 5 * 60, + "1h": 60 * 60, +} + +@dataclass +class IncrementalCounter: + """Счётчик, который умеет: + • `+=` – увеличивает внутренний счётчик на 1 + • `last_5min`, `last_hour`, `total` – сколько было увеличений + за последние 5 минут, 1 час и за всё время соответственно + """ + + # Внутренний счётчик (сумма всех увеличений) + _total: int = 0 + # История – deque из timestamps (float) когда происходил инкремент + _history: Deque[float] = field(default_factory=deque, init=False) + + # ---------- Оператор += ---------- + def __iadd__(self, other): + """ + При любом `+=` увеличиваем счётчик на 1. + Возвращаем self, чтобы поддерживать цепочку выражений. + """ + # Счётчик всегда +1, игнорируем `other` + self._total += 1 + # Храним только время события + self._history.append(time.monotonic()) + # Удаляем слишком старые элементы (самый длинный интервал = 1h) + self._purge_old_entries() + return self + + # ---------- Свойства для статистики ---------- + @property + def total(self) -> int: + """Общее количество прибавлений.""" + return self._total + + @property + def last_5min(self) -> int: + """Сколько прибавлений было за последние 5 минут.""" + return self._count_in_window(WINDOWS["5min"]) + + @property + def last_hour(self) -> int: + """Сколько прибавлений было за последний час.""" + return self._count_in_window(WINDOWS["1h"]) + + # ---------- Вспомогательные методы ---------- + def _purge_old_entries(self) -> None: + """Удаляем из deque все записи старше 1 часа.""" + cutoff = time.monotonic() - WINDOWS["1h"] + while self._history and self._history[0] < cutoff: + self._history.popleft() + + def _count_in_window(self, seconds: float) -> int: + """Подсчёт, сколько событий попадает в заданный интервал.""" + cutoff = time.monotonic() - seconds + # Удаляем старые элементы, которые уже не нужны + while self._history and self._history[0] < cutoff: + self._history.popleft() + return len(self._history) + + # ---------- Пользовательский интерфейс ---------- + def __repr__(self): + return ( + f"" + ) \ No newline at end of file diff --git a/modules/ui/__init__.py b/modules/ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/ui/gui/MainWindow.py b/modules/ui/gui/MainWindow.py new file mode 100644 index 0000000..0041fbe --- /dev/null +++ b/modules/ui/gui/MainWindow.py @@ -0,0 +1,131 @@ +import os.path +from pkgutil import extend_path + +import flet as ft +from Workspaces import * + +MainWindowDarkTheme = { + 'logo_large_path': os.path.join('assets', 'logo_dark_large.png'), + 'logo_small_path': os.path.join('assets', 'logo_dark_small.png'), + 'background_color': "#1f1e23", + 'text_color': "#ffffff", + 'accent_color': "#7B1FA2", + 'icon': ft.Icons.NIGHTLIGHT_ROUNDED +} + +MainWindowLightTheme = { + 'logo_large_path': os.path.join('assets', 'logo_light_large.png'), + 'logo_small_path': os.path.join('assets', 'logo_light_small.png'), + 'background_color': "#ffffff", + 'text_color': "#1f1e23", + 'accent_color': "#9C27B0", + 'icon': ft.Icons.SUNNY +} + +class MainWindow: + def __init__(self): + self.title = 'Vaiola' + self.themes = [MainWindowDarkTheme, MainWindowLightTheme] + self.active_theme_number = 0 + self.active_theme = self.themes[self.active_theme_number] + + self.side_bar = MainWindowSideBar(self) + self.active_workspace = self.side_bar.get_active_workspace() + self.page: ft.Page | None = None + + def build(self, page: ft.Page): + self.page = page + page.clean() + page.title = self.title + page.padding = 0 + page.bgcolor = self.active_theme['background_color'] + self.active_workspace = self.side_bar.get_active_workspace() + layout = ft.Row(controls=[self.side_bar.build(), ft.VerticalDivider(thickness=4, ), self.active_workspace.build()], + spacing=0, + vertical_alignment=ft.CrossAxisAlignment.START, + alignment=ft.MainAxisAlignment.START, + ) + + page.add(layout) + + def set_theme(self): + self.active_theme_number += 1 + if self.active_theme_number >= len(self.themes): self.active_theme_number = 0 + self.active_theme = self.themes[self.active_theme_number] + self.side_bar.set_theme(self.active_theme) + self.rebuild() + + def rebuild(self): + self.build(self.page) + self.page.update() + + +class MainWindowSideBar: + logo_dark_large_path = os.path.join('assets', 'logo_dark_large.png') + logo_dark_small_path = os.path.join('assets', 'logo_dark_small.png') + logo_light_large_path = os.path.join('assets', 'logo_light_large.png') + logo_light_small_path = os.path.join('assets', 'logo_light_small.png') + + def __init__(self, parent): + self.tabs: list[MainWindowSideBarTab] = list() + self.active_tab: MainWindowSideBarTab = MainWindowSideBarTab() + self.extended = True + self.theme = parent.active_theme + self.parent = parent + + def set_theme(self, theme): self.theme = theme + + def get_active_workspace(self) -> Workspace: return self.active_tab.get_active_workspace() + + def build(self) -> ft.Container: + logo = ft.Container( + content=ft.Image( + src=self.theme['logo_large_path'] if self.extended else self.theme['logo_small_path'], + width=200 if self.extended else 60, + fit=ft.ImageFit.CONTAIN + ), + width=200 if self.extended else 60, + # on_click + ) + + theme_button = ft.Button( + content=ft.Row( + controls=[ + ft.Icon(self.theme['icon'], color=self.theme['background_color']), + ft.Text('Переключить тему', color=self.theme['background_color']) + ] if self.extended else [ + ft.Icon(self.theme['icon'], color=self.theme['background_color']) + + ] + ), + bgcolor=self.theme['text_color'], + on_click=self.switch_theme + + + ) + settings = ft.Container(content=theme_button, padding=8) + + + + layout = ft.Column( + controls=[logo, ft.Text('Область вкладок', color=self.theme['text_color']), settings] + ) + return ft.Container(content=layout, width=200 if self.extended else 60,) + + def switch_theme(self, e): + self.parent.set_theme() + + +class MainWindowSideBarTab: + def __init__(self): + self.workspace = Workspace() + + def get_active_workspace(self) -> Workspace: + return self.workspace + + + + +if __name__ == "__main__": + ft.app(target=MainWindow().build) + diff --git a/modules/ui/gui/MainWindowSideMenu.py b/modules/ui/gui/MainWindowSideMenu.py new file mode 100644 index 0000000..33cf857 --- /dev/null +++ b/modules/ui/gui/MainWindowSideMenu.py @@ -0,0 +1,98 @@ +# side_menu.py +import os.path + +import flet as ft + +class NavPanel: + """Пустая панель навигации. Добавьте свои пункты позже.""" + def __init__(self, page: ft.Page): + self.page = page + + def build(self) -> ft.Container: + """Возвращаем контейнер с кнопками навигации.""" + # Пример пунктов – замените/добавьте свои + return ft.Container( + padding=ft.padding.symmetric(vertical=10, horizontal=5), + content=ft.Column( + controls=[ + ft.TextButton( + text="Главная", + icon=ft.Icons.HOME, + on_click=lambda e: self.page.views.append(ft.View("/home")), + style=ft.ButtonStyle(overlay_color=ft.Colors.GREY_200) + ), + ft.TextButton( + text="Настройки", + icon=ft.Icons.SETTINGS, + on_click=lambda e: self.page.views.append(ft.View("/settings")), + style=ft.ButtonStyle(overlay_color=ft.Colors.GREY_200) + ), + ], + spacing=5, + ), + ) + +class SideMenu: + """ + Класс, представляющий боковое меню. + На данный момент оно «пустое», но можно легко добавить пункты в будущем. + """ + def __init__(self): + # Любые начальные данные можно хранить здесь + self.width = 200 # ширина меню + self.bgcolor = ft.Colors.SURFACE + self.logo_path = os.path.join('assets', 'side_menu_logo_dark.png') + + def build(self, page: ft.Page) -> ft.Container: + """ + Возвращает контейнер, который можно вставить в страницу. + """ + logo = ft.Image( + src=self.logo_path, + width=self.width, # растягиваем до ширины меню + fit=ft.ImageFit.CONTAIN, # сохраняем пропорции + # height может быть не задан; Flet будет авто‑подбирать + ) + + # 2️⃣ Панель навигации + nav_panel = NavPanel(page).build() + + # 3️⃣ Кнопка‑тогглер темы + def toggle_theme(e): + # Переключаем режим и обновляем страницу + page.theme_mode = ( + ft.ThemeMode.DARK if page.theme_mode == ft.ThemeMode.LIGHT + else ft.ThemeMode.LIGHT + ) + page.update() + + toggle_btn = ft.TextButton( + text="Тёмная тема" if page.theme_mode == ft.ThemeMode.LIGHT else "Светлая тема", + icon=ft.Icons.BOOKMARK, + on_click=toggle_theme, + style=ft.ButtonStyle( + padding=ft.padding.all(10), + alignment=ft.alignment.center_left + ) + ) + + # 4️⃣ Ставим всё в колонку с выравниванием по краям + return ft.Container( + width=self.width, + bgcolor=self.bgcolor, + padding=ft.padding.symmetric(vertical=15, horizontal=10), + content=ft.Column( + controls=[ + logo, + ft.Divider(height=15), + nav_panel, + ft.Divider(height=15), + ft.Container( + content=toggle_btn, + alignment=ft.alignment.bottom_left + ), + ], + spacing=10, + alignment=ft.MainAxisAlignment.START, + ), + ) \ No newline at end of file diff --git a/modules/ui/gui/Sample.py b/modules/ui/gui/Sample.py new file mode 100644 index 0000000..581dfd4 --- /dev/null +++ b/modules/ui/gui/Sample.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Flet‑приложение: боковая панель + рабочая область +(с учётом всех уточнений: исчезновение текста при сворачивании, +акцент только на активной группе/вкладке, иконка‑кнопка смены темы, +фиолетовый акцентный цвет). +""" + +import flet as ft + +# ---------- Параметры изображений ---------- +LOGO_LIGHT_COLLAPSED = "logo_light_small.png" # светлый логотип, «маленький» +LOGO_LIGHT_EXPANDED = "logo_light_large.png" # светлый логотип, «большой» +LOGO_DARK_COLLAPSED = "logo_dark_small.png" # тёмный логотип, «маленький» +LOGO_DARK_EXPANDED = "logo_dark_large.png" # тёмный логотип, «большой» + +# ---------- Цвета ---------- +LIGHT_BG = "#e0f7fa" +DARK_BG = "#263238" +LIGHT_ACC = "#9C27B0" # фиолетовый 500 +DARK_ACC = "#7B1FA2" # фиолетовый 700 + +# ---------- Вспомогательные функции ---------- +def status_lamp(color: str) -> ft.Icon: + return ft.Icon(ft.Icons.CIRCLE, size=12, color=color) + +def group_icon(name: str) -> ft.Icon: + mapping = { + "Repository": ft.Icons.ARCHIVE, + "Environment": ft.Icons.FOLDER, + "Civit": ft.Icons.HEXAGON, + "Tasks": ft.Icons.DOWNLOAD, + } + return ft.Icon(mapping.get(name, ft.Icons.FOLDER), size=20) + +def tab_icon() -> ft.Icon: + return ft.Icon(ft.Icons.FILE_COPY, size=30) + +# ---------- Главная функция ---------- +def main(page: ft.Page): + page.title = "Flet Sidebar Demo" + page.vertical_alignment = ft.MainAxisAlignment.START + page.horizontal_alignment = ft.CrossAxisAlignment.START + + # ----- Состояния ----- + is_expanded = True + + group_expanded = { + "Repository": True, + "Environment": False, + "Civit": False, + "Tasks": False, + } + + selected_group: str | None = None + selected_tab_name: str | None = None + + selected_tab = ft.Text(value="Выберите вкладку", size=24, weight=ft.FontWeight.W_400) + + # ----- Логотип ----- + def get_logo_path() -> str: + if page.theme_mode == ft.ThemeMode.LIGHT: + return LOGO_LIGHT_EXPANDED if is_expanded else LOGO_LIGHT_COLLAPSED + else: + return LOGO_DARK_EXPANDED if is_expanded else LOGO_DARK_COLLAPSED + + # ----- Панель навигации ----- + sidebar = ft.Container( + width=200 if is_expanded else 60, + bgcolor=ft.Colors.SURFACE, + padding=ft.padding.all(8), + content=ft.Column(spacing=8, controls=[]), + ) + + # ----- Рабочая область ----- + main_area = ft.Container( + expand=True, + padding=ft.padding.all(20), + content=ft.Container( + alignment=ft.alignment.center, + content=selected_tab + ) + ) + + # ----- Макет ----- + page.add(ft.Row(controls=[sidebar, main_area], expand=True)) + + # ----- Пересоздание боковой панели ----- + def rebuild_sidebar(): + """Пересоздаёт содержимое боковой панели.""" + controls = [] + + # 1. Логотип (с кликом) + logo_img = ft.Image( + src=get_logo_path(), + width=50 if not is_expanded else 150, + height=50 if not is_expanded else 150, + fit=ft.ImageFit.CONTAIN + ) + logo_container = ft.Container( + content=logo_img, + on_click=lambda e: toggle_sidebar() + ) + controls.append(logo_container) + + # 2. Группы вкладок + groups = { + "Repository": ["Create", "Upload"], + "Environment": ["Create", "Upload", "Install"], + "Civit": ["Initialize", "Overview", "Selection"], + "Tasks": ["Upload"], + } + + for grp_name, tabs in groups.items(): + controls.append(build_group(grp_name, tabs)) + + # 3. Кнопка смены темы (только иконка) + controls.append(ft.Container(height=20)) + theme_icon = ft.Icons.SUNNY if page.theme_mode == ft.ThemeMode.LIGHT else ft.Icons.NIGHTLIGHT_ROUNDED + theme_btn = ft.IconButton( + icon=theme_icon, + on_click=lambda e: toggle_theme() + ) + controls.append(theme_btn) + + sidebar.content.controls = controls + page.update() + + # ----- Группа + подменю ----- + def build_group(name: str, tabs: list[str]) -> ft.Container: + """Создаёт одну группу с подменю.""" + # Фон заголовка – только для активной группы + header_bg = LIGHT_ACC if selected_group == name else ft.Colors.TRANSPARENT + + # 1️⃣ Первый ряд: статус‑лампочка, иконка, название группы + title_row = ft.Row( + controls=[ + status_lamp("#ffeb3b"), + group_icon(name), + ft.Text(name, weight=ft.FontWeight.BOLD, color=ft.Colors.ON_PRIMARY) + ], + alignment=ft.MainAxisAlignment.START, + vertical_alignment=ft.CrossAxisAlignment.CENTER, + spacing=8, + ) + + # 2️⃣ Второй ряд: подстрока (отображается только при развёрнутой панели) + subtitle_row = ft.Row( + controls=[ + ft.Text("Подстрока", size=10, color=ft.Colors.GREY) + ], + alignment=ft.MainAxisAlignment.START, + vertical_alignment=ft.CrossAxisAlignment.CENTER, + spacing=8, + ) + + header_content = ft.Column( + controls=[title_row] + ([subtitle_row] if is_expanded else []), + spacing=2, + ) + + header = ft.Container( + padding=ft.padding.only(left=8, right=8, top=4, bottom=4), + bgcolor=header_bg, + border_radius=8, + content=header_content, + on_click=lambda e: toggle_group(name), + ) + + # Список вкладок + tab_items = [] + for tab_name in tabs: + icon = tab_icon() + title = ft.Text(tab_name, color=ft.Colors.ON_SURFACE_VARIANT) + if selected_tab_name == tab_name: + icon.color = LIGHT_ACC if page.theme_mode == ft.ThemeMode.LIGHT else DARK_ACC + title.color = LIGHT_ACC if page.theme_mode == ft.ThemeMode.LIGHT else DARK_ACC + + row = ft.Row( + controls=[icon], + alignment=ft.MainAxisAlignment.START, + vertical_alignment=ft.CrossAxisAlignment.CENTER, + spacing=8 + ) + if is_expanded: + row.controls.append(title) + + item = ft.Container( + content=row, + padding=ft.padding.only(left=16), + on_click=lambda e, t=tab_name, g=name: select_tab(g, t) + ) + tab_items.append(item) + + sublist = ft.Container( + content=ft.Column(controls=tab_items, spacing=0), + height=0 if not group_expanded[name] else len(tabs) * 48, + ) + + return ft.Column( + controls=[header, sublist], + spacing=4 + ) + + # ----- События ----- + def toggle_sidebar(): + nonlocal is_expanded + is_expanded = not is_expanded + sidebar.width = 200 if is_expanded else 80 + rebuild_sidebar() + + def toggle_group(name: str): + group_expanded[name] = not group_expanded[name] + rebuild_sidebar() + + def select_tab(group: str, tab: str): + nonlocal selected_group, selected_tab_name + selected_group = group + selected_tab_name = tab + selected_tab.value = f"{tab}\n(Icon + text)" + rebuild_sidebar() + + def toggle_theme(): + if page.theme_mode == ft.ThemeMode.LIGHT: + page.theme_mode = ft.ThemeMode.DARK + page.bgcolor = DARK_BG + else: + page.theme_mode = ft.ThemeMode.LIGHT + page.bgcolor = LIGHT_BG + rebuild_sidebar() + page.update() + + # ----- Инициализация ----- + page.bgcolor = LIGHT_BG + rebuild_sidebar() + + +# ---------- Запуск ---------- +if __name__ == "__main__": + ft.app(target=main) \ No newline at end of file diff --git a/modules/ui/gui/Workspaces.py b/modules/ui/gui/Workspaces.py new file mode 100644 index 0000000..9dc3dc4 --- /dev/null +++ b/modules/ui/gui/Workspaces.py @@ -0,0 +1,7 @@ +import flet as ft + +class Workspace: + def __init__(self): pass + + def build(self) -> ft.Container: + return ft.Container(content=ft.Text("Выберете вкладку")) diff --git a/modules/ui/gui/__init__.py b/modules/ui/gui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 9f40479..02ac313 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ colorama -requests \ No newline at end of file +requests +netifaces +ping3 +flet \ No newline at end of file diff --git a/shell/Handlers/CivitHandler.py b/shell/Handlers/CivitHandler.py index a28cfc6..555e362 100644 --- a/shell/Handlers/CivitHandler.py +++ b/shell/Handlers/CivitHandler.py @@ -1,6 +1,10 @@ import json +import os.path +from modules.civit.Civit import Civit +from modules.civit.datamodel import Creator, Tag, Model from modules.civit.fetch import Fetch +from modules.shared.DatabaseSqlite import SQLiteDatabase from shell.Handlers.ABS import Handler from modules.civit.client import Client @@ -11,25 +15,52 @@ class CivitHandler(Handler): def __init__(self): super().__init__() self.forwarding_table: dict[str, Handler] = { - 'fetch': FetchHandler(self) + 'fetch': FetchHandler(self), + 'save': SaveHandler(self) } self.handle_table: dict = { 'init': self._init, } - self.client: Client | None = None + self.client: Civit | None = None def _init(self, command: list[str], pos=0): - keys, args = self.parse_arguments(command[pos:], ['path', 'key']) + keys, args = self.parse_arguments(command[pos:], ['path', 'key', 'dbpath']) self._check_arg(keys, 'path') - self.client = Client(keys['path'], keys['key']) + if not keys['dbpath']: keys['dbpath'] = keys['path'] + db = SQLiteDatabase(path=keys['dbpath'], name='data') + self.client = Civit(db, path=keys['path'], api_key=keys['key']) self.succeed = True +class SaveHandler(Handler): + def __init__(self, parent): + super().__init__() + self.parent: CivitHandler = parent + self.forwarding_table: dict[str, Handler] = { + } + self.handle_table: dict = { + 'creators': self._creators, + 'tags': self._tags, + 'models': self._models, + 'images': self._images, + } + + def _creators(self, command: list[str], pos=0): + self.parent.client.from_fetch('creator', Creator) + self.succeed = True + + def _tags(self, command: list[str], pos=0): + self.parent.client.from_fetch('tag', Tag) + self.succeed = True + + def _models(self, command: list[str], pos=0): + self.parent.client.from_fetch('model', Model) + self.succeed = True + + def _images(self, command: list[str], pos=0): raise NotImplemented + class FetchHandler(Handler): - - - def __init__(self, parent): super().__init__() self.parent: CivitHandler = parent @@ -51,7 +82,7 @@ class FetchHandler(Handler): keys, args = self.parse_arguments(command[pos:], ['entity']) self._check_arg(keys, 'entity') - res = Fetch.load(self.parent.client, keys['entity']) + res = Fetch.load(self.parent.client.client, keys['entity']) for r in res: print(r) self.succeed = True @@ -59,24 +90,24 @@ class FetchHandler(Handler): def _creators_raw(self, command: list[str], pos=0): keys, args = self.parse_arguments(command[pos:], ['page', 'limit', 'query']) - res = self.parent.client.get_creators_raw(page=keys['page'], limit=keys['limit'], query=keys['query']) + res = self.parent.client.client.get_creators_raw(page=keys['page'], limit=keys['limit'], query=keys['query']) print(res) self.succeed = True def _creators(self, command: list[str], pos=0): - res = Fetch.creators(self.parent.client) + res = self.parent.client.fetcher.creators() for r in res: print(r) self.succeed = True def _tags_raw(self, command: list[str], pos=0): keys, args = self.parse_arguments(command[pos:], ['page', 'limit', 'query']) - res = self.parent.client.get_tags_raw(page=keys['page'], limit=keys['limit'], query=keys['query']) + res = self.parent.client.client.get_tags_raw(page=keys['page'], limit=keys['limit'], query=keys['query']) print(res) self.succeed = True def _tags(self, command: list[str], pos=0): - res = Fetch.tags(self.parent.client) + res = self.parent.client.fetcher.tags() for r in res: print(r) self.succeed = True @@ -87,7 +118,7 @@ class FetchHandler(Handler): def _images(self, command: list[str], pos=0): keys, args = self.parse_arguments(command[pos:], ['start']) - res = Fetch.images(self.parent.client, start_with=keys['start']) + res = Fetch.images(self.parent.client.client, start_with=keys['start']) for r in res: print(r) self.succeed = True @@ -101,7 +132,7 @@ class FetchHandler(Handler): keys, args = self.parse_arguments(command[pos:], ['entity', 'top', 'dump']) self._check_arg(keys, 'entity') if keys['entity'] in entities: - res = Fetch.datamodel(self.parent.client, entities[keys['entity']], keys['top'] or 10) + res = Fetch.datamodel(self.parent.client.client, entities[keys['entity']], keys['top'] or 10) print(json.dumps(res, indent=2, ensure_ascii=False)) if keys['dump']: with open(keys['dump'], 'w') as f: diff --git a/shell/Handlers/ModelSpaceHandler.py b/shell/Handlers/ModelSpaceHandler.py index 5dbcd10..abe4590 100644 --- a/shell/Handlers/ModelSpaceHandler.py +++ b/shell/Handlers/ModelSpaceHandler.py @@ -56,8 +56,14 @@ class ModelSpaceHandler(Handler): def _pull_civit(self, command: list[str], pos=0): keys, args = self.parse_arguments(command[pos:], ['model', 'version', 'file']) - self._check_arg(keys, 'model') - global_repo.model_sub_repo.pull_civit_package(self.client, keys['model'], keys['version'], keys['file']) + + if keys['model']: + global_repo.model_sub_repo.pull_civit_package(self.client, keys['model'], keys['version'], keys['file']) + else: + while True: + model = input("Model ID:") + global_repo.model_sub_repo.pull_civit_package(self.client, model) + self.succeed = True @@ -91,6 +97,7 @@ class ModelSpaceHandler(Handler): # def _create(self, command: list[str], pos = 0): # keys, args = self.parse_arguments(command[pos:], ['env', 'path', 'python']) + self.succeed = True def _install(self, command: list[str], pos = 0): keys, args = self.parse_arguments(command[pos:], ['answer'])