george/PrivateBin
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

started script for storage backend migrations

Browse Source 
todo: GCS

added GCS, no GLOBALS, two methods for saving pastes and comments

use GLOBALS for verbosity again

added getAllPastes() to all storage providers

moved to bin, added --delete options, make use of $store->getAllPastes()

added --delete-* options to help

longopts without -- *sigh*

fixed arguments

drop singleton behaviour to allow multiple backends of the same type simultaneously

remove singleton from Model, collapse loop in migrate.php

comments is not indexed

tests without data singleton

fix

exit if scandir() fails

extended meta doc
This commit is contained in:
Felix J. Ogris
2022-10-28 01:01:02 +02:00
parent d5104a1d63
commit 9a61e8fd48
22 changed files with 658 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
lib/Data/GoogleCloudStorage.php
View File

@@ -14,54 +14,43 @@ class GoogleCloudStorage extends AbstractData
* GCS client
*
* @access private
* @static
* @var StorageClient
*/
private static $_client = null;
private $_client = null;
/**
* GCS bucket
*
* @access private
* @static
* @var Bucket
*/
private static $_bucket = null;
private $_bucket = null;
/**
* object prefix
*
* @access private
* @static
* @var string
*/
private static $_prefix = 'pastes';
private $_prefix = 'pastes';
/**
* bucket acl type
*
* @access private
* @static
* @var bool
*/
private static $_uniformacl = false;
private $_uniformacl = false;
/**
* returns a Google Cloud Storage data backend.
* instantiantes a new Google Cloud Storage data backend.
*
* @access public
* @static
* @param array $options
* @return GoogleCloudStorage
* @return
*/
public static function getInstance(array $options)
public function __construct(array $options)
{
// if needed initialize the singleton
if (!(self::$_instance instanceof self)) {
self::$_instance = new self;
}
$bucket = null;
if (getenv('PRIVATEBIN_GCS_BUCKET')) {
$bucket = getenv('PRIVATEBIN_GCS_BUCKET');
}
@@ -69,24 +58,20 @@ class GoogleCloudStorage extends AbstractData
$bucket = $options['bucket'];
}
if (is_array($options) && array_key_exists('prefix', $options)) {
self::$_prefix = $options['prefix'];
$this->_prefix = $options['prefix'];
}
if (is_array($options) && array_key_exists('uniformacl', $options)) {
self::$_uniformacl = $options['uniformacl'];
$this->_uniformacl = $options['uniformacl'];
}
if (empty(self::$_client)) {
self::$_client = class_exists('StorageClientStub', false) ?
new \StorageClientStub(array()) :
new StorageClient(array('suppressKeyFileNotice' => true));
}
self::$_bucket = self::$_client->bucket($bucket);
return self::$_instance;
$this->_client = class_exists('StorageClientStub', false) ?
new \StorageClientStub(array()) :
new StorageClient(array('suppressKeyFileNotice' => true));
$this->_bucket = $this->_client->bucket($bucket);
}
/**
* returns the google storage object key for $pasteid in self::$_bucket.
* returns the google storage object key for $pasteid in $this->_bucket.
*
* @access private
* @param $pasteid string to get the key for
@@ -94,14 +79,14 @@ class GoogleCloudStorage extends AbstractData
*/
private function _getKey($pasteid)
{
if (self::$_prefix != '') {
return self::$_prefix . '/' . $pasteid;
if ($this->_prefix != '') {
return $this->_prefix . '/' . $pasteid;
}
return $pasteid;
}
/**
* Uploads the payload in the self::$_bucket under the specified key.
* Uploads the payload in the $this->_bucket under the specified key.
* The entire payload is stored as a JSON document. The metadata is replicated
* as the GCS object's metadata except for the fields attachment, attachmentname
* and salt.
@@ -126,12 +111,12 @@ class GoogleCloudStorage extends AbstractData
'metadata' => $metadata,
),
);
if (!self::$_uniformacl) {
if (!$this->_uniformacl) {
$data['predefinedAcl'] = 'private';
}
self::$_bucket->upload(Json::encode($payload), $data);
$this->_bucket->upload(Json::encode($payload), $data);
} catch (Exception $e) {
error_log('failed to upload ' . $key . ' to ' . self::$_bucket->name() . ', ' .
error_log('failed to upload ' . $key . ' to ' . $this->_bucket->name() . ', ' .
trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
return false;
}
@@ -156,13 +141,13 @@ class GoogleCloudStorage extends AbstractData
public function read($pasteid)
{
try {
$o = self::$_bucket->object($this->_getKey($pasteid));
$o = $this->_bucket->object($this->_getKey($pasteid));
$data = $o->downloadAsString();
return Json::decode($data);
} catch (NotFoundException $e) {
return false;
} catch (Exception $e) {
error_log('failed to read ' . $pasteid . ' from ' . self::$_bucket->name() . ', ' .
error_log('failed to read ' . $pasteid . ' from ' . $this->_bucket->name() . ', ' .
trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
return false;
}
@@ -176,9 +161,9 @@ class GoogleCloudStorage extends AbstractData
$name = $this->_getKey($pasteid);
try {
foreach (self::$_bucket->objects(array('prefix' => $name . '/discussion/')) as $comment) {
foreach ($this->_bucket->objects(array('prefix' => $name . '/discussion/')) as $comment) {
try {
self::$_bucket->object($comment->name())->delete();
$this->_bucket->object($comment->name())->delete();
} catch (NotFoundException $e) {
// ignore if already deleted.
}
@@ -188,7 +173,7 @@ class GoogleCloudStorage extends AbstractData
}
try {
self::$_bucket->object($name)->delete();
$this->_bucket->object($name)->delete();
} catch (NotFoundException $e) {
// ignore if already deleted
}
@@ -199,7 +184,7 @@ class GoogleCloudStorage extends AbstractData
*/
public function exists($pasteid)
{
$o = self::$_bucket->object($this->_getKey($pasteid));
$o = $this->_bucket->object($this->_getKey($pasteid));
return $o->exists();
}
@@ -223,8 +208,8 @@ class GoogleCloudStorage extends AbstractData
$comments = array();
$prefix = $this->_getKey($pasteid) . '/discussion/';
try {
foreach (self::$_bucket->objects(array('prefix' => $prefix)) as $key) {
$comment = JSON::decode(self::$_bucket->object($key->name())->downloadAsString());
foreach ($this->_bucket->objects(array('prefix' => $prefix)) as $key) {
$comment = JSON::decode($this->_bucket->object($key->name())->downloadAsString());
$comment['id'] = basename($key->name());
$slot = $this->getOpenSlot($comments, (int) $comment['meta']['created']);
$comments[$slot] = $comment;
@@ -241,7 +226,7 @@ class GoogleCloudStorage extends AbstractData
public function existsComment($pasteid, $parentid, $commentid)
{
$name = $this->_getKey($pasteid) . '/discussion/' . $parentid . '/' . $commentid;
$o = self::$_bucket->object($name);
$o = $this->_bucket->object($name);
return $o->exists();
}
@@ -252,7 +237,7 @@ class GoogleCloudStorage extends AbstractData
{
$path = 'config/' . $namespace;
try {
foreach (self::$_bucket->objects(array('prefix' => $path)) as $object) {
foreach ($this->_bucket->objects(array('prefix' => $path)) as $object) {
$name = $object->name();
if (strlen($name) > strlen($path) && substr($name, strlen($path), 1) !== '/') {
continue;
@@ -300,12 +285,12 @@ class GoogleCloudStorage extends AbstractData
'metadata' => $metadata,
),
);
if (!self::$_uniformacl) {
if (!$this->_uniformacl) {
$data['predefinedAcl'] = 'private';
}
self::$_bucket->upload($value, $data);
$this->_bucket->upload($value, $data);
} catch (Exception $e) {
error_log('failed to set key ' . $key . ' to ' . self::$_bucket->name() . ', ' .
error_log('failed to set key ' . $key . ' to ' . $this->_bucket->name() . ', ' .
trim(preg_replace('/\s\s+/', ' ', $e->getMessage())));
return false;
}
@@ -323,7 +308,7 @@ class GoogleCloudStorage extends AbstractData
$key = 'config/' . $namespace . '/' . $key;
}
try {
$o = self::$_bucket->object($key);
$o = $this->_bucket->object($key);
return $o->downloadAsString();
} catch (NotFoundException $e) {
return '';
@@ -338,12 +323,12 @@ class GoogleCloudStorage extends AbstractData
$expired = array();
$now = time();
$prefix = self::$_prefix;
$prefix = $this->_prefix;
if ($prefix != '') {
$prefix .= '/';
}
try {
foreach (self::$_bucket->objects(array('prefix' => $prefix)) as $object) {
foreach ($this->_bucket->objects(array('prefix' => $prefix)) as $object) {
$metadata = $object->info()['metadata'];
if ($metadata != null && array_key_exists('expire_date', $metadata)) {
$expire_at = intval($metadata['expire_date']);
@@ -361,4 +346,28 @@ class GoogleCloudStorage extends AbstractData
}
return $expired;
}
/**
* @inheritDoc
*/
public function getAllPastes()
{
$pastes = array();
$prefix = $this->_prefix;
if ($prefix != '') {
$prefix .= '/';
}
try {
foreach ($this->_bucket->objects(array('prefix' => $prefix)) as $object) {
$candidate = substr($object->name(), strlen($prefix));
if (strpos($candidate, "/") === false) {
$pastes[] = $candidate;
}
}
} catch (NotFoundException $e) {
// no objects in the bucket yet
}
return $pastes;
}
}