HDK/hyperf-admin
1
0
Fork 0
mirror of http://124.126.16.154:8888/singularity/hyperf-admin.git synced 2026-01-15 05:55:08 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

perf: 文件存储迁移到 filesystem

Browse Source
This commit is contained in:
daodao97
2020-10-15 17:14:03 +08:00
parent 1d993ace73
commit 01e9633e02
12 changed files with 611 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
composer.json
View File

@@ -23,7 +23,7 @@
"hyperf/crontab": "~2.0.0",
"hyperf/database": "~2.0.0",
"hyperf/db-connection": "~2.0.0",
"hyperf/filesystem": "~2.0.0",
"hyperf/filesystem": "^2.0",
"hyperf/framework": "~2.0.0",
"hyperf/guzzle": "~2.0.0",
"hyperf/http-server": "~2.0.0",
@@ -36,6 +36,7 @@
"hyperf/snowflake": "~2.0.0",
"hyperf/validation": "~2.0.0",
"nette/php-generator": "^3.4",
"xxtime/flysystem-aliyun-oss": "^1.5",
"yadakhov/insert-on-duplicate-key": "^1.2",
"zoujingli/ip2region": "^1.0"
},

1
docs/_sidebar.md
View File

@@ -14,6 +14,7 @@
* [远程脚手架](backend/remote_scaffold.md)
* [通用配置](backend/common-config.md)
* [辅助函数](backend/functions.md)
* [文件上传](backend/file.md)
* 业务组件
* [业务组件介绍](backend/components/business/desc.md)
* [DevTools-开发者工具](backend/components/business/dev-tools.md)

37
docs/backend/file.md Normal file
View File

@@ -0,0 +1,37 @@
## 文件上传
文件的处理统一使用 `hyperf/filesystem`, 请先阅读其文档 [biu~~](https://hyperf.wiki/2.0/#/zh-cn/filesystem)
### 文件的上传
表单的控件中可以指定 `存储介质`, `可见性` 等.
```php
'form' => [
'avatar|用户头像' => [
'type' => 'image',
'rule' => 'string',
'readonly' => true,
'props' => [
'bucket' => 'aliyuncs', // 指定存储的storage, 可选详见 config/autoload/file.php storage
'private' => true, // 是否为私有
]
],
]
```
### 两个快捷方法
1. `move_local_file_to_filesystem($local_file_path, $save_file_path, $private = false, $bucket = 'aliyuncs', $update_when_exist = true)`
将本地文件通过 `filesystem` 指定的介质来存储
2. `filesystem_private_url($save_file_path, $timeout = 60, $bucket = 'aliyuncs')`
获取私有文件的临时访问链接
### 提示
如果存储方式为 `本地`, 我们 增加了 `file.storage.local.cdn` 这个配置项, 用于生成可用的访问链接.
如果使用的其他存储介质, 请记得安装相应扩展包.
!> 当前仅对 `本地`, `阿里云oss` 做了适配, 因为没有其他元的账号..., 如果您当前使用的存储介质(如: 腾讯云), 请反馈给我们, 如能提供测试账号验证下更好.

4
docs/backend/functions.md
View File

@@ -52,9 +52,9 @@
#### register_route
#### move_local_file_to_oss
#### move_local_file_to_filesystem
#### oss_private_url
#### filesystem_private_url
#### call_self_api

16
src/admin/src/Controller/UploadController.php
View File

@@ -1,7 +1,6 @@
<?php
namespace HyperfAdmin\Admin\Controller;
use OSS\Core\OssException;
use HyperfAdmin\BaseUtils\Constants\ErrorCode;
use HyperfAdmin\BaseUtils\Log;
use HyperfAdmin\BaseUtils\Scaffold\Controller\Controller;
@@ -10,17 +9,20 @@ class UploadController extends Controller
{
public function image()
{
$bucket = $this->request->input('bucket', 'local');
$private = $this->request->input('private', false);
$file = $this->request->file('file');
if (!$file->isValid()) {
return $this->fail(ErrorCode::CODE_ERR_PARAM);
}
$tmp_file = $file->toArray()['tmp_file'];
$md5_filename = md5_file($tmp_file);
$path = '1/' . date('Ym') . '/' . $md5_filename . '.' . $file->getExtension();
$bucket = $this->request->input('bucket', 'aliyuncs');
$private = $this->request->input('private', false);
try {
$uploaded = move_local_file_to_oss($tmp_file, $path, $private, $bucket);
$uploaded = move_local_file_to_filesystem($tmp_file, $path, $private, $bucket);
if ($uploaded === false) {
return $this->fail(ErrorCode::CODE_ERR_SERVER, '上传失败');
}
@@ -35,7 +37,7 @@ class UploadController extends Controller
];
return $this->success($info);
} catch (OssException $e) {
} catch (\Exception $e) {
Log::get('upload')->error($e->getMessage());
return $this->fail(ErrorCode::CODE_ERR_SERVER, $e->getMessage());
@@ -45,10 +47,12 @@ class UploadController extends Controller
public function privateFileUrl()
{
$oss_path = $this->request->input('key');
$bucket = $this->request->input('storage', config('file.default'));
if (!$oss_path) {
return $this->fail(ErrorCode::CODE_ERR_PARAM);
}
$private_url = oss_private_url($oss_path);
$private_url = filesystem_private_url($oss_path, MINUTE * 5, $bucket);
if (!$private_url) {
return $this->fail(ErrorCode::CODE_ERR_SYSTEM);
}

4
src/admin/src/Service/ExportService.php
View File

@@ -129,8 +129,8 @@ class ExportService
])->save();
$query['_page'] += 1;
}
$bucket = config('storager.export_service', config('storager.default'));
$info = move_local_file_to_oss($file_path, '1/export_task/' . $file_name, true, $bucket);
$bucket = config('file.export_storage', config('file.default'));
$info = move_local_file_to_filesystem($file_path, '1/export_task/' . $file_name, true, $bucket);
if($info) {
$task->fill([
'status' => ExportTasks::STATUS_SUCCESS,

3
src/base-utils/composer.json
View File

@@ -60,6 +60,9 @@
"src/Helper/common.php",
"src/Helper/array.php",
"src/Helper/system.php"
],
"classmap": [
"src/classmap"
]
},
"autoload-dev": {

92
src/base-utils/src/Helper/system.php
View File

@@ -1,14 +1,14 @@
<?php
use Hyperf\ExceptionHandler\Formatter\FormatterInterface;
use Hyperf\Filesystem\FilesystemFactory;
use Hyperf\HttpServer\Router\DispatcherFactory;
use Hyperf\HttpServer\Router\Router;
use Hyperf\Server\ServerFactory;
use Hyperf\Utils\Str;
use OSS\Core\OssException;
use HyperfAdmin\BaseUtils\AliyunOSS;
use HyperfAdmin\BaseUtils\Guzzle;
use HyperfAdmin\BaseUtils\Log;
use League\Flysystem\AdapterInterface;
if (!function_exists('server')) {
function server()
@@ -68,41 +68,81 @@ if(!function_exists('register_route')) {
}
}
if(!function_exists('move_local_file_to_oss')) {
function move_local_file_to_oss($local_file_path, $oss_file_path, $private = false, $bucket = 'aliyuncs')
if (!function_exists('move_local_file_to_filesystem')) {
function move_local_file_to_filesystem($local_file_path, $save_file_path, $private = false, $bucket = 'aliyuncs', $update_when_exist = true)
{
/** @var AliyunOSS $oss */
$oss = make(AliyunOSS::class, ['bucket' => $bucket]);
$filesystem = make(FilesystemFactory::class)->get($bucket);
try {
$method = $private ? 'uploadPrivateFile' : 'uploadFile';
$oss->$method($oss_file_path, $local_file_path);
$file_path = config('storager.aliyuncs.cdn') . '/' . $oss_file_path;
if($private) {
$file_path = oss_private_url($oss_file_path, MINUTE * 5, $bucket);
$stream = fopen($local_file_path, 'r');
$has = $filesystem->has($save_file_path);
if (!$has) {
$filesystem->writeStream($save_file_path, $stream);
}
if ($has && $update_when_exist) {
$filesystem->updateStream($save_file_path, $stream);
}
if ($private) {
$filesystem->setVisibility($save_file_path, AdapterInterface::VISIBILITY_PRIVATE);
}
$meta = $filesystem->getMetadata($save_file_path);
switch (config("file.storage.{$bucket}.driver")) {
case \Hyperf\Filesystem\Adapter\LocalAdapterFactory::class:
$file_path = config("file.storage.{$bucket}.cdn") . $meta['path'];
break;
case \Hyperf\Filesystem\Adapter\AliyunOssAdapterFactory::class;
$file_path = $meta['info']['url'];
break;
// TODO 更多 filesystem 渠道
default:
$file_path = $save_file_path;
}
if ($private) {
$file_path = filesystem_private_url($save_file_path, MINUTE * 5, $bucket);
}
return [
'file_path' => $file_path,
'path' => 'oss/' . $oss_file_path,
'file_path' => $file_path, // 直接可访问的完整链接
'path' => $save_file_path, // 资源的存储路径
];
} catch (OssException $exception) {
Log::get('move_local_file_to_oss')->error($exception->getMessage());
} catch (\Exception $exception) {
Log::get('move_local_file_to_filesystem')->error($exception->getMessage());
return false;
}
}
}
if(!function_exists('oss_private_url')) {
function oss_private_url($oss_file_path, $timeout = 60, $bucket = 'aliyuncs')
if (!function_exists('filesystem_private_url')) {
function filesystem_private_url($save_file_path, $timeout = 60, $bucket = 'aliyuncs')
{
/** @var AliyunOSS $oss */
$oss = make(AliyunOSS::class, ['bucket' => $bucket]);
$key = preg_replace('@^oss/@', '', $oss_file_path);
if (!$save_file_path) {
return false;
}
$filesystem = make(FilesystemFactory::class)->get($bucket);
if (Str::startsWith($save_file_path, 'http')) {
$save_file_path = parse_url($save_file_path)['path'];
$parts = explode('/', $save_file_path);
unset($parts[0]);
$save_file_path = implode('/', $parts);
}
$save_file_path = preg_replace('@^oss/@', '', $save_file_path);
try {
return str_replace('-internal', '', $oss->getSignUrl($key, $timeout));
} catch (OssException $exception) {
Log::get('oss_private_url')->error($exception->getMessage());
switch (config("file.storage.{$bucket}.driver")) {
case \Hyperf\Filesystem\Adapter\LocalAdapterFactory::class:
return config("file.storage.{$bucket}.cdn") . $save_file_path;
break;
case \Hyperf\Filesystem\Adapter\AliyunOssAdapterFactory::class;
$adapter = $filesystem->getAdapter();
if (method_exists($adapter, 'signUrl')) {
return $adapter->signUrl($save_file_path, $timeout);
}
break;
// TODO 更多 filesystem 渠道
default:
return $save_file_path;
}
} catch (\Exception $exception) {
Log::get('filesystem_private_url')->error($exception->getMessage());
return false;
}
@@ -164,11 +204,7 @@ if(!function_exists('get_sub_dir')) {
function get_sub_dir($dir, $exclude = [])
{
$paths = [];
$dirs = \Symfony\Component\Finder\Finder::create()
->in($dir)
->depth('<1')
->exclude((array)$exclude)
->directories();
$dirs = \Symfony\Component\Finder\Finder::create()->in($dir)->depth('<1')->exclude((array)$exclude)->directories();
/** @var SplFileInfo $dir */
foreach ($dirs as $dir) {
$paths[] = $dir->getRealPath();

6
src/base-utils/src/Middleware/HttpLogMiddleware.php
View File

@@ -49,10 +49,14 @@ class HttpLogMiddleware implements MiddlewareInterface
case Dispatcher::FOUND:
if(!in_array($path, $uriEx)) {
$response_content = mb_substr($response->getBody()->getContents(), 0, 125);
$request_content = $request->getBody()->getContents();
if (preg_match('/filename=.(.*)/', $request_content, $m)) {
$request_content = urlencode($m[1]);
}
$msg = [
'referer' => $referer,
'uri' => $request->getRequestTarget(),
'request' => $request->getBody()->getContents(),
'request' => $request_content,
'use_time' => 1000 * (microtime(true) - $start_time),
'response' => $response_content,
];

2
src/base-utils/src/Scaffold/Controller/AbstractController.php
View File

@@ -265,7 +265,7 @@ abstract class AbstractController extends Controller
if (is_callable($item['render'])) {
$each[$item['field']] = $item['render']($each[$item['field']] ?? null, $each);
} elseif (is_string($item['render']) && method_exists($this, $item['render'])) {
$each[$item['field']] = $this->$item['render']($each[$item['field']] ?? null, $each);
$each[$item['field']] = $this->{$item['render']}($each[$item['field']] ?? null, $each);
}
unset($each);
}

451
src/base-utils/src/classmap/OssAdapter.php Normal file
View File

@@ -0,0 +1,451 @@
<?php
/**
* 增加此文件的目的是因为原生的类中缺少更多 AliyunOSS 底层的更多方法
* 增加的方法有
* signUrl 获取文件的私有访问地址
*/
namespace Xxtime\Flysystem\Aliyun;
use League\Flysystem\Adapter\AbstractAdapter;
use League\Flysystem\Config;
use League\Flysystem\Util;
use OSS\OssClient;
use Exception;
class OssAdapter extends AbstractAdapter
{
/**
* @var Supports
*/
public $supports;
/**
* @var OssClient
*/
private $oss;
/**
* @var AliYun bucket
*/
private $bucket;
/**
* @var string
*/
private $endpoint = 'oss-cn-hangzhou.aliyuncs.com';
/**
* OssAdapter constructor.
* @param array $config
* @throws Exception
*/
public function __construct($config = [])
{
$isCName = false;
$token = null;
$this->supports = new Supports();
try {
$this->bucket = $config['bucket'];
empty($config['endpoint']) ? null : $this->endpoint = $config['endpoint'];
empty($config['timeout']) ? $config['timeout'] = 3600 : null;
empty($config['connectTimeout']) ? $config['connectTimeout'] = 10 : null;
if (!empty($config['isCName'])) {
$isCName = true;
}
if (!empty($config['token'])) {
$token = $config['token'];
}
$this->oss = new OssClient(
$config['accessId'], $config['accessSecret'], $this->endpoint, $isCName, $token
);
$this->oss->setTimeout($config['timeout']);
$this->oss->setConnectTimeout($config['connectTimeout']);
} catch (Exception $e) {
throw $e;
}
}
/**
* Write a new file.
*
* @param string $path
* @param string $contents
* @param Config $config Config object
*
* @return array|false false on failure file meta data on success
*/
public function write($path, $contents, Config $config)
{
$result = $this->oss->putObject($this->bucket, $path, $contents, $this->getOssOptions($config));
$this->supports->setFlashData($result);
return $result;
}
/**
* Write a new file using a stream.
*
* @param string $path
* @param resource $resource
* @param Config $config Config object
*
* @return array|false false on failure file meta data on success
*/
public function writeStream($path, $resource, Config $config)
{
if (!is_resource($resource)) {
return false;
}
$i = 0;
$bufferSize = 1000000; // 1M
while (!feof($resource)) {
if (false === $buffer = fread($resource, $block = $bufferSize)) {
return false;
}
$position = $i * $bufferSize;
$size = $this->oss->appendObject($this->bucket, $path, $buffer, $position, $this->getOssOptions($config));
$i++;
}
fclose($resource);
return true;
}
/**
* Update a file.
*
* @param string $path
* @param string $contents
* @param Config $config Config object
*
* @return array|false false on failure file meta data on success
*/
public function update($path, $contents, Config $config)
{
$result = $this->oss->putObject($this->bucket, $path, $contents, $this->getOssOptions($config));
$this->supports->setFlashData($result);
return $result;
}
/**
* Update a file using a stream.
*
* @param string $path
* @param resource $resource
* @param Config $config Config object
*
* @return array|false false on failure file meta data on success
*/
public function updateStream($path, $resource, Config $config)
{
$result = $this->write($path, stream_get_contents($resource), $config);
if (is_resource($resource)) {
fclose($resource);
}
return $result;
}
/**
* Rename a file.
*
* @param string $path
* @param string $newpath
*
* @return bool
*/
public function rename($path, $newpath)
{
$this->oss->copyObject($this->bucket, $path, $this->bucket, $newpath);
$this->oss->deleteObject($this->bucket, $path);
return true;
}
/**
* Copy a file.
*
* @param string $path
* @param string $newpath
*
* @return bool
*/
public function copy($path, $newpath)
{
$this->oss->copyObject($this->bucket, $path, $this->bucket, $newpath);
return true;
}
/**
* Delete a file.
*
* @param string $path
*
* @return bool
*/
public function delete($path)
{
$this->oss->deleteObject($this->bucket, $path);
return true;
}
/**
* Delete a directory.
*
* @param string $dirname
*
* @return bool
*/
public function deleteDir($dirname)
{
$lists = $this->listContents($dirname, true);
if (!$lists) {
return false;
}
$objectList = [];
foreach ($lists as $value) {
$objectList[] = $value['path'];
}
$this->oss->deleteObjects($this->bucket, $objectList);
return true;
}
/**
* Create a directory.
*
* @param string $dirname directory name
* @param Config $config
*
* @return array|false
*/
public function createDir($dirname, Config $config)
{
$this->oss->createObjectDir($this->bucket, $dirname);
return true;
}
/**
* Set the visibility for a file.
*
* @param string $path
* @param string $visibility
*
* @return array|false file meta data
*
* Aliyun OSS ACL value: 'default', 'private', 'public-read', 'public-read-write'
*/
public function setVisibility($path, $visibility)
{
$this->oss->putObjectAcl(
$this->bucket,
$path,
($visibility == 'public') ? 'public-read' : 'private'
);
return true;
}
/**
* Check whether a file exists.
*
* @param string $path
*
* @return array|bool|null
*/
public function has($path)
{
return $this->oss->doesObjectExist($this->bucket, $path);
}
/**
* Read a file.
*
* @param string $path
*
* @return array|false
*/
public function read($path)
{
return [
'contents' => $this->oss->getObject($this->bucket, $path)
];
}
/**
* Read a file as a stream.
*
* @param string $path
*
* @return array|false
*/
public function readStream($path)
{
$resource = 'http://' . $this->bucket . '.' . $this->endpoint . '/' . $path;
return [
'stream' => $resource = fopen($resource, 'r')
];
}
/**
* List contents of a directory.
*
* @param string $directory
* @param bool $recursive
*
* @return array
*/
public function listContents($directory = '', $recursive = false)
{
$directory = rtrim($directory, '\\/');
$result = [];
$nextMarker = '';
while (true) {
// max-keys 用于限定此次返回object的最大数，如果不设定，默认为100，max-keys取值不能大于1000。
// prefix 限定返回的object key必须以prefix作为前缀。注意使用prefix查询时，返回的key中仍会包含prefix。
// delimiter是一个用于对Object名字进行分组的字符。所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组元素
// marker 用户设定结果从marker之后按字母排序的第一个开始返回。
$options = [
'max-keys' => 1000,
'prefix' => $directory . '/',
'delimiter' => '/',
'marker' => $nextMarker,
];
$res = $this->oss->listObjects($this->bucket, $options);
// 得到nextMarker，从上一次$res读到的最后一个文件的下一个文件开始继续获取文件列表
$nextMarker = $res->getNextMarker();
$prefixList = $res->getPrefixList(); // 目录列表
$objectList = $res->getObjectList(); // 文件列表
if ($prefixList) {
foreach ($prefixList as $value) {
$result[] = [
'type' => 'dir',
'path' => $value->getPrefix()
];
if ($recursive) {
$result = array_merge($result, $this->listContents($value->getPrefix(), $recursive));
}
}
}
if ($objectList) {
foreach ($objectList as $value) {
if (($value->getSize() === 0) && ($value->getKey() === $directory . '/')) {
continue;
}
$result[] = [
'type' => 'file',
'path' => $value->getKey(),
'timestamp' => strtotime($value->getLastModified()),
'size' => $value->getSize()
];
}
}
if ($nextMarker === '') {
break;
}
}
return $result;
}
/**
* Get all the meta data of a file or directory.
*
* @param string $path
*
* @return array|false
*/
public function getMetadata($path)
{
return $this->oss->getObjectMeta($this->bucket, $path);
}
/**
* Get the size of a file.
*
* @param string $path
*
* @return array|false
*/
public function getSize($path)
{
$response = $this->oss->getObjectMeta($this->bucket, $path);
return [
'size' => $response['content-length']
];
}
/**
* Get the mimetype of a file.
*
* @param string $path
*
* @return array|false
*/
public function getMimetype($path)
{
$response = $this->oss->getObjectMeta($this->bucket, $path);
return [
'mimetype' => $response['content-type']
];
}
/**
* Get the timestamp of a file.
*
* @param string $path
*
* @return array|false
*/
public function getTimestamp($path)
{
$response = $this->oss->getObjectMeta($this->bucket, $path);
return [
'timestamp' => $response['last-modified']
];
}
/**
* Get the visibility of a file.
*
* @param string $path
*
* @return array|false
*/
public function getVisibility($path)
{
$response = $this->oss->getObjectAcl($this->bucket, $path);
return [
'visibility' => $response,
];
}
/**
* Get OSS Options
* @param Config $config
* @return array
*/
private function getOssOptions(Config $config)
{
$options = [];
if ($config->has("headers")) {
$options['headers'] = $config->get("headers");
}
if ($config->has("Content-Type")) {
$options["Content-Type"] = $config->get("Content-Type");
}
if ($config->has("Content-Md5")) {
$options["Content-Md5"] = $config->get("Content-Md5");
$options["checkmd5"] = false;
}
return $options;
}
public function signUrl($path, $timeout = 60)
{
return $this->oss->signUrl($this->bucket, $path, $timeout);
}
}

10
src/cron-center/src/ClassJobAbstract.php
View File

@@ -18,26 +18,24 @@ abstract class ClassJobAbstract
$this->job = $crontab;
$this->jobManager = make(CronManager::class);
$this->state = $this->jobManager->getJobState($crontab->getId());
$this->logger = logger();
$this->logger = logger()->get('cron_center');
}
public function run($params = [])
{
$func = function () use ($params) {
try {
$this->beforeAction($params);
$this->handle($params);
$result = $this->handle($params);
$this->afterAction($params);
return $result;
} catch (\Throwable $throwable) {
$this->onError($throwable);
return false;
} finally {
$this->onComplete();
return $this->evaluate();
}
};
return $func();
}
public function beforeAction($params = [])