Browse Source

#123: The process command (which is now bt-queue:process to avoid conflicts with Laravel's default queue namespace) now uses the S3 storage to retrieve images

tags/v2.2.0-beta.1
Andy Heathershaw 7 months ago
parent
commit
95e79f2d28
7 changed files with 133 additions and 13 deletions
  1. +48
    -0
      app/AlbumSources/AmazonS3Source.php
  2. +16
    -0
      app/AlbumSources/IAnalysisQueueSource.php
  3. +9
    -8
      app/Console/Commands/ProcessQueueCommand.php
  4. +12
    -0
      app/Http/Controllers/Admin/PhotoController.php
  5. +9
    -4
      app/Services/PhotoService.php
  6. +38
    -0
      database/migrations/2019_07_13_075909_add_new_album_id_column.php
  7. +1
    -1
      resources/systemd/blue-twilight-mq.service

+ 48
- 0
app/AlbumSources/AmazonS3Source.php View File

@@ -20,6 +20,30 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
// The delete routine will have already removed all photos
}

/**
* Deletes a photo to be analysed from the storage source
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return void
*/
public function deleteItemFromAnalysisQueue($queueToken, $fileName)
{
$fileToDelete = $this->getPathToAnalysisQueueItem($queueToken, $fileName);

try
{
$this->getClient()->deleteObject([
'Bucket' => $this->configuration->container_name,
'Key' => $fileToDelete
]);
}
catch (ClientErrorResponseException $ex)
{
// Don't worry if the file no longer exists
Log::warning('Failed deleting image from S3.', ['error' => $ex->getMessage(), 'path' => $fileToDelete]);
}
}

/**
* Deletes a thumbnail file for a photo.
* @param Photo $photo Photo to delete the thumbnail from.
@@ -44,6 +68,25 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
}
}

/**
* Downloads a photo to be analysed from the storage source to a temporary file
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return string Path to the photo that was downloaded
*/
public function fetchItemFromAnalysisQueue($queueToken, $fileName)
{
$tempFile = tempnam(sys_get_temp_dir(), 'BlueTwilight_');

$this->getClient()->getObject([
'Bucket' => $this->configuration->container_name,
'Key' => $this->getPathToAnalysisQueueItem($queueToken, $fileName),
'SaveAs' => $tempFile
]);

return $tempFile;
}

/**
* Fetches the contents of a thumbnail for a photo.
* @param Photo $photo Photo to fetch the thumbnail for.
@@ -155,6 +198,11 @@ class AmazonS3Source extends AlbumSourceBase implements IAlbumSource, IAnalysisQ
return sprintf('analysis-queue/%s', $queueToken);
}

private function getPathToAnalysisQueueItem($queueToken, $fileName)
{
return sprintf('%s/%s', $this->getPathToAnalysisQueue($queueToken), $fileName);
}

private function getPathToPhoto(Photo $photo, $thumbnail = null)
{
return sprintf(


+ 16
- 0
app/AlbumSources/IAnalysisQueueSource.php View File

@@ -6,6 +6,22 @@ use App\Storage;

interface IAnalysisQueueSource
{
/**
* Deletes a photo to be analysed from the storage source
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return void
*/
function deleteItemFromAnalysisQueue($queueToken, $fileName);

/**
* Downloads a photo to be analysed from the storage source to a temporary file
* @param string $queueToken Queue token holding the photo
* @param string $fileName Filename of the photo to download
* @return string Path to the photo that was downloaded
*/
function fetchItemFromAnalysisQueue($queueToken, $fileName);

/**
* @param Storage $configuration
* @return mixed


+ 9
- 8
app/Console/Commands/ProcessQueueCommand.php View File

@@ -2,6 +2,7 @@

namespace App\Console\Commands;

use App\Album;
use App\Facade\UserConfig;
use App\Photo;
use App\QueueItem;
@@ -10,6 +11,7 @@ use App\Services\RabbitMQService;
use App\User;
use App\UserActivity;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\App;

class ProcessQueueCommand extends Command
{
@@ -18,7 +20,7 @@ class ProcessQueueCommand extends Command
*
* @var string
*/
protected $signature = 'queue:process';
protected $signature = 'bt-queue:process';

/**
* The console command description.
@@ -181,21 +183,20 @@ class ProcessQueueCommand extends Command
switch (strtolower($action))
{
/* This needs a bit more work - we need to also store the new album ID in the queue_items table */
/*case 'change_album':
case 'change_album':
if ($user->can('change-metadata', $photo))
{
$newAlbumId = intval($request->get('new-album-id'));
if ($newAlbumId == $photo->album_id)
$newAlbum = Album::where('id', intval($queueItem->new_album_id))->first();
if (is_null($newAlbum) || !$user->can('upload-photos', $newAlbum))
{
// Photo already belongs to this album, don't move
$this->output->writeln('Target album does not exist or user does not have permission.');
return;
}

$newAlbum = $this->loadAlbum($newAlbumId, 'upload-photos');
$this->output->writeln(sprintf('Moving photo to album \'%s\'', $newAlbum->name));
$photoService->changeAlbum($newAlbum);
$changed = true;
}
break;*/
break;

case 'delete':
if ($user->can('delete', $photo))


+ 12
- 0
app/Http/Controllers/Admin/PhotoController.php View File

@@ -601,6 +601,18 @@ class PhotoController extends Controller
'user_id' => $this->getUser()->id,
'queued_at' => new \DateTime()
]);

if (strtolower($action) == 'change_album')
{
$queueItem->new_album_id = intval($request->get('new-album-id'));
$newAlbumId = intval($request->get('new-album-id'));
if ($newAlbumId == $photo->album_id)
{
// Photo already belongs to this album, don't move
continue;
}
}

$queueItem->save();

$rabbitmq = new RabbitMQService();


+ 9
- 4
app/Services/PhotoService.php View File

@@ -4,6 +4,8 @@ namespace App\Services;

use App\Album;
use App\AlbumSources\IAlbumSource;
use App\AlbumSources\IAnalysisQueueSource;
use App\Helpers\AnalysisQueueHelper;
use App\Helpers\FileHelper;
use App\Helpers\ImageHelper;
use App\Helpers\MiscHelper;
@@ -52,8 +54,10 @@ class PhotoService

public function analyse($queueToken, $isReanalyse = false)
{
$queuePath = FileHelper::getQueuePath($queueToken);
$photoFile = join(DIRECTORY_SEPARATOR, [$queuePath, $this->photo->storage_file_name]);
/** @var IAnalysisQueueSource $analysisQueueStorage */
$analysisQueueStorage = AnalysisQueueHelper::getStorageQueueSource();

$photoFile = $analysisQueueStorage->fetchItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name);

try
{
@@ -143,10 +147,11 @@ class PhotoService
}
finally
{
// Remove the temporary file
@unlink($photoFile);

// If the queue directory is now empty, get rid of it
FileHelper::deleteIfEmpty($queuePath);
// Remove from the storage
$analysisQueueStorage->deleteItemFromAnalysisQueue($queueToken, $this->photo->storage_file_name);
}
}



+ 38
- 0
database/migrations/2019_07_13_075909_add_new_album_id_column.php View File

@@ -0,0 +1,38 @@
<?php

use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;

class AddNewAlbumIdColumn extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('user_activity', function (Blueprint $table)
{
$table->unsignedInteger('new_album_id')->nullable(true);

$table->foreign('new_album_id')
->references('id')->on('albums')
->onDelete('cascade');
});
}

/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('user_activity', function (Blueprint $table)
{
$table->dropColumn('new_album_id');
});
}
}

+ 1
- 1
resources/systemd/blue-twilight-mq.service View File

@@ -3,7 +3,7 @@ Description=Blue Twilight Processing Queue Runner

[Service]
WorkingDirectory=/data/www/blue-twilight.andysh.dev/
ExecStart=/usr/bin/php artisan queue:process
ExecStart=/usr/bin/php artisan bt-queue:process
Restart=on-failure
User=www-data
Group=www-data


Loading…
Cancel
Save