george/PrivateBin
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

started script for storage backend migrations

Browse Source 
todo: GCS

added GCS, no GLOBALS, two methods for saving pastes and comments

use GLOBALS for verbosity again

added getAllPastes() to all storage providers

moved to bin, added --delete options, make use of $store->getAllPastes()

added --delete-* options to help

longopts without -- *sigh*

fixed arguments

drop singleton behaviour to allow multiple backends of the same type simultaneously

remove singleton from Model, collapse loop in migrate.php

comments is not indexed

tests without data singleton

fix

exit if scandir() fails

extended meta doc
This commit is contained in:
Felix J. Ogris
2022-10-28 01:01:02 +02:00
parent d5104a1d63
commit 9a61e8fd48
22 changed files with 658 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
lib/Data/Filesystem.php
View File

@@ -40,33 +40,26 @@ class Filesystem extends AbstractData
* path in which to persist something
*
* @access private
* @static
* @var string
*/
private static $_path = 'data';
private $_path = 'data';
/**
* get instance of singleton
* instantiates a new Filesystem data backend
*
* @access public
* @static
* @param array $options
* @return Filesystem
* @return
*/
public static function getInstance(array $options)
public function __construct(array $options)
{
// if needed initialize the singleton
if (!(self::$_instance instanceof self)) {
self::$_instance = new self;
}
// if given update the data directory
if (
is_array($options) &&
array_key_exists('dir', $options)
) {
self::$_path = $options['dir'];
$this->_path = $options['dir'];
}
return self::$_instance;
}
/**
@@ -79,7 +72,7 @@ class Filesystem extends AbstractData
*/
public function create($pasteid, array $paste)
{
$storagedir = self::_dataid2path($pasteid);
$storagedir = $this->_dataid2path($pasteid);
$file = $storagedir . $pasteid . '.php';
if (is_file($file)) {
return false;
@@ -87,7 +80,7 @@ class Filesystem extends AbstractData
if (!is_dir($storagedir)) {
mkdir($storagedir, 0700, true);
}
return self::_store($file, $paste);
return $this->_store($file, $paste);
}
/**
@@ -101,7 +94,7 @@ class Filesystem extends AbstractData
{
if (
!$this->exists($pasteid) ||
!$paste = self::_get(self::_dataid2path($pasteid) . $pasteid . '.php')
!$paste = $this->_get($this->_dataid2path($pasteid) . $pasteid . '.php')
) {
return false;
}
@@ -116,7 +109,7 @@ class Filesystem extends AbstractData
*/
public function delete($pasteid)
{
$pastedir = self::_dataid2path($pasteid);
$pastedir = $this->_dataid2path($pasteid);
if (is_dir($pastedir)) {
// Delete the paste itself.
if (is_file($pastedir . $pasteid . '.php')) {
@@ -124,7 +117,7 @@ class Filesystem extends AbstractData
}
// Delete discussion if it exists.
$discdir = self::_dataid2discussionpath($pasteid);
$discdir = $this->_dataid2discussionpath($pasteid);
if (is_dir($discdir)) {
// Delete all files in discussion directory
$dir = dir($discdir);
@@ -148,20 +141,20 @@ class Filesystem extends AbstractData
*/
public function exists($pasteid)
{
$basePath = self::_dataid2path($pasteid) . $pasteid;
$basePath = $this->_dataid2path($pasteid) . $pasteid;
$pastePath = $basePath . '.php';
// convert to PHP protected files if needed
if (is_readable($basePath)) {
self::_prependRename($basePath, $pastePath);
$this->_prependRename($basePath, $pastePath);
// convert comments, too
$discdir = self::_dataid2discussionpath($pasteid);
$discdir = $this->_dataid2discussionpath($pasteid);
if (is_dir($discdir)) {
$dir = dir($discdir);
while (false !== ($filename = $dir->read())) {
if (substr($filename, -4) !== '.php' && strlen($filename) >= 16) {
$commentFilename = $discdir . $filename . '.php';
self::_prependRename($discdir . $filename, $commentFilename);
$this->_prependRename($discdir . $filename, $commentFilename);
}
}
$dir->close();
@@ -182,7 +175,7 @@ class Filesystem extends AbstractData
*/
public function createComment($pasteid, $parentid, $commentid, array $comment)
{
$storagedir = self::_dataid2discussionpath($pasteid);
$storagedir = $this->_dataid2discussionpath($pasteid);
$file = $storagedir . $pasteid . '.' . $commentid . '.' . $parentid . '.php';
if (is_file($file)) {
return false;
@@ -190,7 +183,7 @@ class Filesystem extends AbstractData
if (!is_dir($storagedir)) {
mkdir($storagedir, 0700, true);
}
return self::_store($file, $comment);
return $this->_store($file, $comment);
}
/**
@@ -203,7 +196,7 @@ class Filesystem extends AbstractData
public function readComments($pasteid)
{
$comments = array();
$discdir = self::_dataid2discussionpath($pasteid);
$discdir = $this->_dataid2discussionpath($pasteid);
if (is_dir($discdir)) {
$dir = dir($discdir);
while (false !== ($filename = $dir->read())) {
@@ -212,7 +205,7 @@ class Filesystem extends AbstractData
// - commentid is the comment identifier itself.
// - parentid is the comment this comment replies to (It can be pasteid)
if (is_file($discdir . $filename)) {
$comment = self::_get($discdir . $filename);
$comment = $this->_get($discdir . $filename);
$items = explode('.', $filename);
// Add some meta information not contained in file.
$comment['id'] = $items[1];
@@ -243,7 +236,7 @@ class Filesystem extends AbstractData
public function existsComment($pasteid, $parentid, $commentid)
{
return is_file(
self::_dataid2discussionpath($pasteid) .
$this->_dataid2discussionpath($pasteid) .
$pasteid . '.' . $commentid . '.' . $parentid . '.php'
);
}
@@ -261,20 +254,20 @@ class Filesystem extends AbstractData
{
switch ($namespace) {
case 'purge_limiter':
return self::_storeString(
self::$_path . DIRECTORY_SEPARATOR . 'purge_limiter.php',
return $this->_storeString(
$this->_path . DIRECTORY_SEPARATOR . 'purge_limiter.php',
'<?php' . PHP_EOL . '$GLOBALS[\'purge_limiter\'] = ' . $value . ';'
);
case 'salt':
return self::_storeString(
self::$_path . DIRECTORY_SEPARATOR . 'salt.php',
return $this->_storeString(
$this->_path . DIRECTORY_SEPARATOR . 'salt.php',
'<?php # |' . $value . '|'
);
case 'traffic_limiter':
self::$_last_cache[$key] = $value;
return self::_storeString(
self::$_path . DIRECTORY_SEPARATOR . 'traffic_limiter.php',
'<?php' . PHP_EOL . '$GLOBALS[\'traffic_limiter\'] = ' . var_export(self::$_last_cache, true) . ';'
$this->_last_cache[$key] = $value;
return $this->_storeString(
$this->_path . DIRECTORY_SEPARATOR . 'traffic_limiter.php',
'<?php' . PHP_EOL . '$GLOBALS[\'traffic_limiter\'] = ' . var_export($this->_last_cache, true) . ';'
);
}
return false;
@@ -292,14 +285,14 @@ class Filesystem extends AbstractData
{
switch ($namespace) {
case 'purge_limiter':
$file = self::$_path . DIRECTORY_SEPARATOR . 'purge_limiter.php';
$file = $this->_path . DIRECTORY_SEPARATOR . 'purge_limiter.php';
if (is_readable($file)) {
require $file;
return $GLOBALS['purge_limiter'];
}
break;
case 'salt':
$file = self::$_path . DIRECTORY_SEPARATOR . 'salt.php';
$file = $this->_path . DIRECTORY_SEPARATOR . 'salt.php';
if (is_readable($file)) {
$items = explode('|', file_get_contents($file));
if (is_array($items) && count($items) == 3) {
@@ -308,12 +301,12 @@ class Filesystem extends AbstractData
}
break;
case 'traffic_limiter':
$file = self::$_path . DIRECTORY_SEPARATOR . 'traffic_limiter.php';
$file = $this->_path . DIRECTORY_SEPARATOR . 'traffic_limiter.php';
if (is_readable($file)) {
require $file;
self::$_last_cache = $GLOBALS['traffic_limiter'];
if (array_key_exists($key, self::$_last_cache)) {
return self::$_last_cache[$key];
$this->_last_cache = $GLOBALS['traffic_limiter'];
if (array_key_exists($key, $this->_last_cache)) {
return $this->_last_cache[$key];
}
}
break;
@@ -325,11 +318,10 @@ class Filesystem extends AbstractData
* get the data
*
* @access public
* @static
* @param string $filename
* @return array|false $data
*/
private static function _get($filename)
private function _get($filename)
{
return Json::decode(
substr(
@@ -350,7 +342,7 @@ class Filesystem extends AbstractData
{
$pastes = array();
$firstLevel = array_filter(
scandir(self::$_path),
scandir($this->_path),
'PrivateBin\Data\Filesystem::_isFirstLevelDir'
);
if (count($firstLevel) > 0) {
@@ -358,7 +350,7 @@ class Filesystem extends AbstractData
for ($i = 0, $max = $batchsize * 10; $i < $max; ++$i) {
$firstKey = array_rand($firstLevel);
$secondLevel = array_filter(
scandir(self::$_path . DIRECTORY_SEPARATOR . $firstLevel[$firstKey]),
scandir($this->_path . DIRECTORY_SEPARATOR . $firstLevel[$firstKey]),
'PrivateBin\Data\Filesystem::_isSecondLevelDir'
);
@@ -369,7 +361,7 @@ class Filesystem extends AbstractData
}
$secondKey = array_rand($secondLevel);
$path = self::$_path . DIRECTORY_SEPARATOR .
$path = $this->_path . DIRECTORY_SEPARATOR .
$firstLevel[$firstKey] . DIRECTORY_SEPARATOR .
$secondLevel[$secondKey];
if (!is_dir($path)) {
@@ -412,6 +404,46 @@ class Filesystem extends AbstractData
return $pastes;
}
/**
* @inheritDoc
*/
public function getAllPastes()
{
$pastes = array();
$subdirs = scandir($this->_path);
if ($subdirs === false) {
dieerr("Unable to list directory " . $this->_path);
}
$subdirs = preg_grep("/^[^.].$/", $subdirs);
foreach ($subdirs as $subdir) {
$subpath = $this->_path . DIRECTORY_SEPARATOR . $subdir;
$subsubdirs = scandir($subpath);
if ($subsubdirs === false) {
dieerr("Unable to list directory " . $subpath);
}
$subsubdirs = preg_grep("/^[^.].$/", $subsubdirs);
foreach ($subsubdirs as $subsubdir) {
$subsubpath = $subpath . DIRECTORY_SEPARATOR . $subsubdir;
$files = scandir($subsubpath);
if ($files === false) {
dieerr("Unable to list directory " . $subsubpath);
}
$files = preg_grep("/\.php$/", $files);
foreach ($files as $file) {
if (substr($file, 0, 4) === $subdir . $subsubdir) {
$pastes[] = substr($file, 0, strlen($file) - 4);
}
}
}
}
return $pastes;
}
/**
* Convert paste id to storage path.
*
@@ -423,13 +455,12 @@ class Filesystem extends AbstractData
* eg. input 'e3570978f9e4aa90' --> output 'data/e3/57/'
*
* @access private
* @static
* @param string $dataid
* @return string
*/
private static function _dataid2path($dataid)
private function _dataid2path($dataid)
{
return self::$_path . DIRECTORY_SEPARATOR .
return $this->_path . DIRECTORY_SEPARATOR .
substr($dataid, 0, 2) . DIRECTORY_SEPARATOR .
substr($dataid, 2, 2) . DIRECTORY_SEPARATOR;
}
@@ -440,13 +471,12 @@ class Filesystem extends AbstractData
* eg. input 'e3570978f9e4aa90' --> output 'data/e3/57/e3570978f9e4aa90.discussion/'
*
* @access private
* @static
* @param string $dataid
* @return string
*/
private static function _dataid2discussionpath($dataid)
private function _dataid2discussionpath($dataid)
{
return self::_dataid2path($dataid) . $dataid .
return $this->_dataid2path($dataid) . $dataid .
'.discussion' . DIRECTORY_SEPARATOR;
}
@@ -454,25 +484,23 @@ class Filesystem extends AbstractData
* Check that the given element is a valid first level directory.
*
* @access private
* @static
* @param string $element
* @return bool
*/
private static function _isFirstLevelDir($element)
private function _isFirstLevelDir($element)
{
return self::_isSecondLevelDir($element) &&
is_dir(self::$_path . DIRECTORY_SEPARATOR . $element);
return $this->_isSecondLevelDir($element) &&
is_dir($this->_path . DIRECTORY_SEPARATOR . $element);
}
/**
* Check that the given element is a valid second level directory.
*
* @access private
* @static
* @param string $element
* @return bool
*/
private static function _isSecondLevelDir($element)
private function _isSecondLevelDir($element)
{
return (bool) preg_match('/^[a-f0-9]{2}$/', $element);
}
@@ -481,15 +509,14 @@ class Filesystem extends AbstractData
* store the data
*
* @access public
* @static
* @param string $filename
* @param array $data
* @return bool
*/
private static function _store($filename, array $data)
private function _store($filename, array $data)
{
try {
return self::_storeString(
return $this->_storeString(
$filename,
self::PROTECTION_LINE . PHP_EOL . Json::encode($data)
);
@@ -502,20 +529,19 @@ class Filesystem extends AbstractData
* store a string
*
* @access public
* @static
* @param string $filename
* @param string $data
* @return bool
*/
private static function _storeString($filename, $data)
private function _storeString($filename, $data)
{
// Create storage directory if it does not exist.
if (!is_dir(self::$_path)) {
if (!@mkdir(self::$_path, 0700)) {
if (!is_dir($this->_path)) {
if (!@mkdir($this->_path, 0700)) {
return false;
}
}
$file = self::$_path . DIRECTORY_SEPARATOR . '.htaccess';
$file = $this->_path . DIRECTORY_SEPARATOR . '.htaccess';
if (!is_file($file)) {
$writtenBytes = 0;
if ($fileCreated = @touch($file)) {
@@ -553,12 +579,11 @@ class Filesystem extends AbstractData
* rename a file, prepending the protection line at the beginning
*
* @access public
* @static
* @param string $srcFile
* @param string $destFile
* @return void
*/
private static function _prependRename($srcFile, $destFile)
private function _prependRename($srcFile, $destFile)
{
// don't overwrite already converted file
if (!is_readable($destFile)) {