diff --git a/api/all_tasks.py b/api/all_tasks.py new file mode 100644 index 0000000000..8e8e03a16d --- /dev/null +++ b/api/all_tasks.py @@ -0,0 +1,90 @@ +import io +import os +import uuid +import zipfile +from datetime import datetime + +import pytz +from django.conf import settings +from django.utils import timezone +from django_q.tasks import AsyncTask, schedule + +import api.util as util +from api.models.long_running_job import LongRunningJob + + +def create_download_job(job_type, user, photos, filename): + job_id = uuid.uuid4() + lrj = LongRunningJob.objects.create( + started_by=user, + job_id=job_id, + queued_at=datetime.now().replace(tzinfo=pytz.utc), + job_type=job_type, + ) + if job_type == LongRunningJob.JOB_DOWNLOAD_PHOTOS: + AsyncTask( + zip_photos_task, job_id=job_id, user=user, photos=photos, filename=filename + ).run() + + lrj.save() + return job_id + + +def zip_photos_task(job_id, user, photos, filename): + lrj = LongRunningJob.objects.get(job_id=job_id) + lrj.started_at = datetime.now().replace(tzinfo=pytz.utc) + count = len(photos) + lrj.result = {"progress": {"current": 0, "target": count}} + lrj.save() + output_directory = os.path.join(settings.MEDIA_ROOT, "zip") + zip_file_name = filename + done_count = 0 + try: + if not os.path.exists(output_directory): + os.mkdir(output_directory) + mf = io.BytesIO() + photos_name = {} + + for photo in photos.values(): + done_count = done_count + 1 + photo_name = os.path.basename(photo.main_file.path) + if photo_name in photos_name: + photos_name[photo_name] = photos_name[photo_name] + 1 + photo_name = str(photos_name[photo_name]) + "-" + photo_name + else: + photos_name[photo_name] = 1 + with zipfile.ZipFile(mf, mode="a", compression=zipfile.ZIP_DEFLATED) as zf: + zf.write(photo.main_file.path, arcname=photo_name) + lrj.result = {"progress": {"current": done_count, "target": count}} + lrj.save() + with open(os.path.join(output_directory, zip_file_name), "wb") as output_file: + output_file.write(mf.getvalue()) + + except Exception as e: + util.logger.error("Error while converting files to zip: {}".format(e)) + + lrj.finished_at = datetime.now().replace(tzinfo=pytz.utc) + lrj.finished = True + lrj.save() + # scheduling a task to delete the zip file after a day + execution_time = timezone.now() + timezone.timedelta(days=1) + schedule("api.all_tasks.delete_zip_file", filename, next_run=execution_time) + return os.path.join(output_directory, zip_file_name) + + +def delete_zip_file(filename): + file_path = os.path.join(settings.MEDIA_ROOT, "zip", filename) + try: + if not os.path.exists(file_path): + util.logger.error( + "Error while deleting file not found at : {}".format(file_path) + ) + return + else: + os.remove(file_path) + util.logger.info("file deleted sucessfully at path : {}".format(file_path)) + return + + except Exception as e: + util.logger.error("Error while deleting file: {}".format(e)) + return e diff --git a/api/batch_jobs.py b/api/batch_jobs.py index 55d2a5ea15..8e2633f223 100644 --- a/api/batch_jobs.py +++ b/api/batch_jobs.py @@ -9,12 +9,11 @@ import api.util as util from api.image_similarity import build_image_similarity_index +from api.ml_models import download_models from api.models.long_running_job import LongRunningJob from api.models.photo import Photo from api.semantic_search.semantic_search import semantic_search_instance -from api.ml_models import download_models - def create_batch_job(job_type, user): job_id = uuid.uuid4() diff --git a/api/face_recognition.py b/api/face_recognition.py index 89e90b6360..4a0d35912a 100644 --- a/api/face_recognition.py +++ b/api/face_recognition.py @@ -1,5 +1,5 @@ -import requests import numpy as np +import requests def get_face_encodings(image_path, known_face_locations): diff --git a/api/geocode/geocode.py b/api/geocode/geocode.py index 0c43bdfc58..93ca2e5724 100644 --- a/api/geocode/geocode.py +++ b/api/geocode/geocode.py @@ -2,6 +2,7 @@ from constance import config as site_config import api.util as util + from .config import get_provider_config, get_provider_parser diff --git a/api/im2txt/build_vocab.py b/api/im2txt/build_vocab.py index c11e85ff4b..357f264ee9 100644 --- a/api/im2txt/build_vocab.py +++ b/api/im2txt/build_vocab.py @@ -32,8 +32,8 @@ def __len__(self): def build_vocab(json, threshold): - from pycocotools.coco import COCO import nltk + from pycocotools.coco import COCO """Build a simple vocabulary wrapper.""" coco = COCO(json) diff --git a/api/im2txt/data_loader.py b/api/im2txt/data_loader.py index ab16536429..702ec8dc0f 100644 --- a/api/im2txt/data_loader.py +++ b/api/im2txt/data_loader.py @@ -1,8 +1,8 @@ import os -from PIL import Image import torch import torch.utils.data as data +from PIL import Image class CocoDataset(data.Dataset): diff --git a/api/im2txt/model.py b/api/im2txt/model.py index 29303b6c69..88a4957b95 100644 --- a/api/im2txt/model.py +++ b/api/im2txt/model.py @@ -1,7 +1,6 @@ import torch import torch.nn as nn import torchvision.models as models -from torch.nn.utils.rnn import pack_padded_sequence class EncoderCNN(nn.Module): diff --git a/api/im2txt/sample.py b/api/im2txt/sample.py index a71fc93a0a..6c23a45a74 100644 --- a/api/im2txt/sample.py +++ b/api/im2txt/sample.py @@ -1,15 +1,14 @@ import os import pickle +import onnxruntime as ort import torch from django.conf import settings +from numpy import asarray from PIL import Image from torchvision import transforms from api.im2txt.model import DecoderRNN, EncoderCNN -import onnxruntime as ort - -from numpy import asarray embed_size = 256 hidden_size = 512 @@ -162,7 +161,7 @@ def generate_caption( return sentence def export_onnx(self, encoder_output_path, decoder_output_path): - from torch.onnx import export, dynamo_export + from torch.onnx import dynamo_export, export self.load_models() @@ -216,10 +215,7 @@ def export_onnx(self, encoder_output_path, decoder_output_path): ".onnx", "_quantizedyn.onnx" ) - from onnxruntime.quantization import ( - quantize_dynamic, - ) - + from onnxruntime.quantization import quantize_dynamic from onnxruntime.quantization.shape_inference import quant_pre_process quant_pre_process_encoder_output_path = encoder_output_path.replace( diff --git a/api/migrations/0055_alter_longrunningjob_job_type.py b/api/migrations/0055_alter_longrunningjob_job_type.py new file mode 100644 index 0000000000..30f10a23da --- /dev/null +++ b/api/migrations/0055_alter_longrunningjob_job_type.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.6 on 2023-10-27 13:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0054_user_cluster_selection_epsilon_user_min_samples"), + ] + + operations = [ + migrations.AlterField( + model_name="longrunningjob", + name="job_type", + field=models.PositiveIntegerField( + choices=[ + (1, "Scan Photos"), + (2, "Generate Event Albums"), + (3, "Regenerate Event Titles"), + (4, "Train Faces"), + (5, "Delete Missing Photos"), + (7, "Scan Faces"), + (6, "Calculate Clip Embeddings"), + (8, "Find Similar Faces"), + (9, "Download Selected Photos"), + ] + ), + ), + ] diff --git a/api/ml_models.py b/api/ml_models.py index 9d3e12268d..39d5f80d64 100644 --- a/api/ml_models.py +++ b/api/ml_models.py @@ -1,13 +1,14 @@ -import pytz +import tarfile from datetime import datetime -from django.conf import settings +from pathlib import Path -from api.models.long_running_job import LongRunningJob -import api.util as util +import pytz import requests -import tarfile -from pathlib import Path from constance import config as site_config +from django.conf import settings + +import api.util as util +from api.models.long_running_job import LongRunningJob class MlTypes: diff --git a/api/models/long_running_job.py b/api/models/long_running_job.py index 9585b8dec1..33c0368d7a 100644 --- a/api/models/long_running_job.py +++ b/api/models/long_running_job.py @@ -18,6 +18,7 @@ class LongRunningJob(models.Model): JOB_CALCULATE_CLIP_EMBEDDINGS = 6 JOB_SCAN_FACES = 7 JOB_CLUSTER_ALL_FACES = 8 + JOB_DOWNLOAD_PHOTOS = 9 JOB_DOWNLOAD_MODELS = 10 JOB_TYPES = ( @@ -29,6 +30,7 @@ class LongRunningJob(models.Model): (JOB_SCAN_FACES, "Scan Faces"), (JOB_CALCULATE_CLIP_EMBEDDINGS, "Calculate Clip Embeddings"), (JOB_CLUSTER_ALL_FACES, "Find Similar Faces"), + (JOB_DOWNLOAD_PHOTOS, "Download Selected Photos"), (JOB_DOWNLOAD_MODELS, "Download Models"), ) diff --git a/api/models/photo.py b/api/models/photo.py index 7a1173ae38..90fe7f5cd0 100644 --- a/api/models/photo.py +++ b/api/models/photo.py @@ -11,15 +11,15 @@ from django.db import models from django.db.models import Q from django.db.utils import IntegrityError -from api.im2txt.sample import Im2txt -from api.face_recognition import get_face_encodings, get_face_locations import api.date_time_extractor as date_time_extractor import api.models import api.util as util from api.exif_tags import Tags +from api.face_recognition import get_face_encodings, get_face_locations from api.geocode import GEOCODE_VERSION from api.geocode.geocode import reverse_geocode +from api.im2txt.sample import Im2txt from api.models.file import File from api.models.user import User, get_deleted_user from api.places365.places365 import place365_instance diff --git a/api/places365/wideresnet.py b/api/places365/wideresnet.py index 8129411970..6b38134b61 100644 --- a/api/places365/wideresnet.py +++ b/api/places365/wideresnet.py @@ -1,7 +1,7 @@ import math import os -import torch.nn as nn +import torch.nn as nn from django.conf import settings model_path = os.path.join(settings.MEDIA_ROOT, "data_models", "resnet18-5c106cde.pth") diff --git a/api/serializers/user.py b/api/serializers/user.py index fbb4a6c11b..e849acb524 100644 --- a/api/serializers/user.py +++ b/api/serializers/user.py @@ -6,10 +6,10 @@ from rest_framework.exceptions import ValidationError from api.batch_jobs import create_batch_job +from api.ml_models import do_all_models_exist from api.models import LongRunningJob, Photo, User from api.serializers.simple import PhotoSuperSimpleSerializer from api.util import logger -from api.ml_models import do_all_models_exist class UserSerializer(serializers.ModelSerializer): diff --git a/api/tests/test_only_photos_or_only_videos.py b/api/tests/test_only_photos_or_only_videos.py index d2ddcf95f9..4e91efa996 100644 --- a/api/tests/test_only_photos_or_only_videos.py +++ b/api/tests/test_only_photos_or_only_videos.py @@ -1,12 +1,9 @@ - -from django.utils import timezone -from unittest.mock import patch from django.test import TestCase +from django.utils import timezone from rest_framework.test import APIClient -from api.models.album_date import AlbumDate -from api.tests.utils import create_test_photos, create_test_user,create_test_photo - +from api.models.album_date import AlbumDate +from api.tests.utils import create_test_photo, create_test_user class OnlyPhotosOrOnlyVideosTest(TestCase): @@ -14,21 +11,18 @@ def setUp(self): self.client = APIClient() self.user = create_test_user() self.client.force_authenticate(user=self.user) - + def test_only_photos(self): now = timezone.now() - photo= create_test_photo( owner=self.user, added_on=now,public=True) + photo = create_test_photo(owner=self.user, added_on=now, public=True) - album=AlbumDate(owner=self.user) - album.id=1 + album = AlbumDate(owner=self.user) + album.id = 1 album.photos.add(photo) album.save() response = self.client.get("/api/albums/date/list?photo=true").url - response =self.client.get(response) + response = self.client.get(response) data = response.json() self.assertEqual(1, len(data["results"])) - - - diff --git a/api/tests/test_zip_list_photos_view_v2.py b/api/tests/test_zip_list_photos_view_v2.py new file mode 100644 index 0000000000..9f6261138d --- /dev/null +++ b/api/tests/test_zip_list_photos_view_v2.py @@ -0,0 +1,36 @@ +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone +from rest_framework.test import APIClient + +from api.models.long_running_job import LongRunningJob +from api.tests.utils import create_test_photos, create_test_user + + +class PhotoListWithoutTimestampTest(TestCase): + def setUp(self): + self.client = APIClient() + self.user = create_test_user() + self.client.force_authenticate(user=self.user) + + @patch("shutil.disk_usage") + def test_download(self, patched_shutil): + # test download function when we have enough storage + patched_shutil.return_value.free = 500000000 + now = timezone.now() + create_test_photos(number_of_photos=1, owner=self.user, added_on=now, size=100) + + response = self.client.get("/api/photos/notimestamp/") + img_hash = response.json()["results"][0]["url"] + datadict = {"owner": self.user, "image_hashes": [img_hash]} + + response_2 = self.client.post("/api/photos/download", data=datadict) + lrr_job = LongRunningJob.objects.all()[0] + self.assertEqual(lrr_job.job_id, response_2.json()["job_id"]) + self.assertEqual(response_2.status_code, 200) + + # test download function when we dont have enough storage + patched_shutil.return_value.free = 0 + response_3 = self.client.post("/api/photos/download", data=datadict) + self.assertEqual(response_3.status_code, 507) diff --git a/api/util.py b/api/util.py index 873ac89b02..e79c885ae3 100644 --- a/api/util.py +++ b/api/util.py @@ -3,8 +3,6 @@ import os.path import exiftool -import requests -from constance import config as site_config from django.conf import settings logger = logging.getLogger("ownphotos") diff --git a/api/views/faces.py b/api/views/faces.py index f4d807b623..b9745c8b55 100644 --- a/api/views/faces.py +++ b/api/views/faces.py @@ -7,8 +7,10 @@ from rest_framework.response import Response from rest_framework.views import APIView +from api.batch_jobs import create_batch_job from api.directory_watcher import scan_faces from api.face_classify import cluster_all_faces +from api.ml_models import do_all_models_exist from api.models import Face, LongRunningJob from api.models.person import Person, get_or_create_person from api.serializers.face import ( @@ -19,8 +21,6 @@ from api.util import logger from api.views.custom_api_view import ListViewSet from api.views.pagination import RegularResultsSetPagination -from api.ml_models import do_all_models_exist -from api.batch_jobs import create_batch_job class ScanFacesView(APIView): diff --git a/api/views/photos.py b/api/views/photos.py index 904a65e215..38cc2a54b5 100755 --- a/api/views/photos.py +++ b/api/views/photos.py @@ -496,4 +496,4 @@ def delete(self, request): if result: return Response(status=status.HTTP_200_OK) else: - return Response(status=status.HTTP_400_BAD_REQUEST) \ No newline at end of file + return Response(status=status.HTTP_400_BAD_REQUEST) diff --git a/api/views/views.py b/api/views/views.py index b0de905e44..8004656522 100644 --- a/api/views/views.py +++ b/api/views/views.py @@ -1,16 +1,13 @@ -import io import os import subprocess import uuid -import zipfile from urllib.parse import quote -from api.batch_jobs import create_batch_job import jsonschema import magic from constance import config as site_config from django.conf import settings -from django.db.models import Q +from django.db.models import Q, Sum from django.http import HttpResponse, HttpResponseForbidden, StreamingHttpResponse from django.utils.decorators import method_decorator from django.utils.encoding import iri_to_uri @@ -24,15 +21,17 @@ from rest_framework_simplejwt.exceptions import TokenError from rest_framework_simplejwt.tokens import AccessToken +from api.all_tasks import create_download_job, delete_zip_file from api.api_util import get_search_term_examples from api.autoalbum import delete_missing_photos +from api.batch_jobs import create_batch_job from api.directory_watcher import scan_photos -from api.models import AlbumUser, Photo, User, LongRunningJob +from api.ml_models import do_all_models_exist +from api.models import AlbumUser, LongRunningJob, Photo, User from api.schemas.site_settings import site_settings_schema from api.serializers.album_user import AlbumUserEditSerializer, AlbumUserListSerializer from api.util import logger from api.views.pagination import StandardResultsSetPagination -from api.ml_models import do_all_models_exist def custom_exception_handler(exc, context): @@ -113,7 +112,6 @@ def post(self, request, format=None): class SetUserAlbumShared(APIView): def post(self, request, format=None): data = dict(request.data) - # print(data) shared = data["shared"] # bool target_user_id = data["target_user_id"] # user pk, int user_album_id = data["album_id"] @@ -370,11 +368,13 @@ def _generate_response(self, photo, path, fname, transcode_videos): path, fname + ".mp4" ) return response + if "faces" in path: response = HttpResponse() response["Content-Type"] = "image/jpg" response["X-Accel-Redirect"] = self._get_protected_media_url(path, fname) return response + if photo.video: # This is probably very slow -> Save the mime type when scanning mime = magic.Magic(mime=True) @@ -427,6 +427,26 @@ def _generate_response(self, photo, path, fname, transcode_videos): ], ) def get(self, request, path, fname, format=None): + if path.lower() == "zip": + jwt = request.COOKIES.get("jwt") + if jwt is not None: + try: + token = AccessToken(jwt) + except TokenError: + return HttpResponseForbidden() + else: + return HttpResponseForbidden() + try: + filename = fname + str(token["user_id"]) + ".zip" + response = HttpResponse() + response["Content-Type"] = "application/x-zip-compressed" + response["X-Accel-Redirect"] = self._get_protected_media_url( + path, filename + ) + return response + except Exception: + return HttpResponseForbidden() + if path.lower() == "avatars": jwt = request.COOKIES.get("jwt") if jwt is not None: @@ -567,33 +587,72 @@ def get(self, request, path, fname, format=None): return HttpResponse(status=404) -class ZipListPhotosView(APIView): - def post(self, request, format=None): +class ZipListPhotosView_V2(APIView): + def post(self, request): + import shutil + + free_storage = shutil.disk_usage("/").free + data = dict(request.data) + if "image_hashes" not in data: + return + photo_query = Photo.objects.filter(owner=self.request.user) + photos = photo_query.in_bulk(data["image_hashes"]) + if len(photos) == 0: + return + total_file_size = photo_query.aggregate(Sum("size"))["size__sum"] or None + if free_storage < total_file_size: + return Response(data={"status": "Insufficient Storage"}, status=507) + file_uuid = uuid.uuid4() + filename = str(str(file_uuid) + str(self.request.user.id) + ".zip") + user = self.request.user + print(user.id) + job_id = create_download_job( + LongRunningJob.JOB_DOWNLOAD_PHOTOS, + user=user, + photos=photos, + filename=filename, + ) + response = {"job_id": job_id, "url": file_uuid} + + return Response(data=response, status=200) + + def get(self, request): + job_id = request.GET["job_id"] + print(job_id) + if job_id is None: + return Response(status=404) try: - data = dict(request.data) - if "image_hashes" not in data: - return - photos = Photo.objects.filter(owner=self.request.user).in_bulk( - data["image_hashes"] - ) - if len(photos) == 0: - return - mf = io.BytesIO() - photos_name = {} - for photo in photos.values(): - photo_name = os.path.basename(photo.main_file.path) - if photo_name in photos_name: - photos_name[photo_name] = photos_name[photo_name] + 1 - photo_name = str(photos_name[photo_name]) + "-" + photo_name - else: - photos_name[photo_name] = 1 - with zipfile.ZipFile( - mf, mode="a", compression=zipfile.ZIP_DEFLATED - ) as zf: - zf.write(photo.main_file.path, arcname=photo_name) - return HttpResponse( - mf.getvalue(), content_type="application/x-zip-compressed" - ) + job = LongRunningJob.objects.get(job_id=job_id) + if job.finished: + return Response(data={"status": "SUCCESS"}, status=200) + elif job.failed: + return Response( + data={"status": "FAILURE", "result": job.result}, status=500 + ) + else: + return Response( + data={"status": "PENDING", "progress": job.result}, status=202 + ) except BaseException as e: logger.error(str(e)) - return HttpResponse(status=404) + return Response(status=404) + + +class DeleteZipView(APIView): + def delete(self, request, fname): + print("hello") + jwt = request.COOKIES.get("jwt") + if jwt is not None: + try: + token = AccessToken(jwt) + except TokenError: + return HttpResponseForbidden() + else: + return HttpResponseForbidden() + filename = fname + str(token["user_id"]) + ".zip" + try: + delete_zip_file(filename) + return Response(status=200) + except BaseException as e: + logger.error(str(e)) + return Response(status=404) diff --git a/librephotos/urls.py b/librephotos/urls.py index d03b0fa56b..2bcb9f1e12 100644 --- a/librephotos/urls.py +++ b/librephotos/urls.py @@ -212,10 +212,15 @@ def post(self, request, *args, **kwargs): views.MediaAccessFullsizeOriginalView.as_view(), name="media", ), + re_path( + r"^api/delete/zip/(?P.*)", + views.DeleteZipView.as_view(), + name="delete-zip", + ), re_path(r"^api/rqavailable/$", jobs.QueueAvailabilityView.as_view()), re_path(r"^api/nextcloud/listdir", nextcloud_views.ListDir.as_view()), re_path(r"^api/nextcloud/scanphotos", nextcloud_views.ScanPhotosView.as_view()), - re_path(r"^api/photos/download", views.ZipListPhotosView.as_view()), + re_path(r"^api/photos/download$", views.ZipListPhotosView_V2.as_view()), re_path(r"^api/timezones", timezone.TimeZoneView.as_view()), re_path(r"api/upload/complete/", upload.UploadPhotosChunkedComplete.as_view()), re_path(r"api/upload/", upload.UploadPhotosChunked.as_view()), diff --git a/service/face_recognition/main.py b/service/face_recognition/main.py index 31f2f609df..d30d32a4d4 100644 --- a/service/face_recognition/main.py +++ b/service/face_recognition/main.py @@ -1,9 +1,9 @@ -import gevent -from flask import Flask, request -from gevent.pywsgi import WSGIServer import face_recognition +import gevent import numpy as np import PIL +from flask import Flask, request +from gevent.pywsgi import WSGIServer app = Flask(__name__)