diff --git a/composer.json b/composer.json index 2fa0bbf..00663a6 100644 --- a/composer.json +++ b/composer.json @@ -23,7 +23,7 @@ "hyperf/crontab": "~2.0.0", "hyperf/database": "~2.0.0", "hyperf/db-connection": "~2.0.0", - "hyperf/filesystem": "~2.0.0", + "hyperf/filesystem": "^2.0", "hyperf/framework": "~2.0.0", "hyperf/guzzle": "~2.0.0", "hyperf/http-server": "~2.0.0", @@ -36,6 +36,7 @@ "hyperf/snowflake": "~2.0.0", "hyperf/validation": "~2.0.0", "nette/php-generator": "^3.4", + "xxtime/flysystem-aliyun-oss": "^1.5", "yadakhov/insert-on-duplicate-key": "^1.2", "zoujingli/ip2region": "^1.0" }, diff --git a/docs/_sidebar.md b/docs/_sidebar.md index b5308c3..9c44d6c 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -14,6 +14,7 @@ * [远程脚手架](backend/remote_scaffold.md) * [通用配置](backend/common-config.md) * [辅助函数](backend/functions.md) + * [文件上传](backend/file.md) * 业务组件 * [业务组件介绍](backend/components/business/desc.md) * [DevTools-开发者工具](backend/components/business/dev-tools.md) diff --git a/docs/backend/file.md b/docs/backend/file.md new file mode 100644 index 0000000..12f9593 --- /dev/null +++ b/docs/backend/file.md @@ -0,0 +1,37 @@ +## 文件上传 + +文件的处理统一使用 `hyperf/filesystem`, 请先阅读其文档 [biu~~](https://hyperf.wiki/2.0/#/zh-cn/filesystem) + +### 文件的上传 + +表单的控件中可以指定 `存储介质`, `可见性` 等. + +```php + 'form' => [ + 'avatar|用户头像' => [ + 'type' => 'image', + 'rule' => 'string', + 'readonly' => true, + 'props' => [ + 'bucket' => 'aliyuncs', // 指定存储的storage, 可选详见 config/autoload/file.php storage + 'private' => true, // 是否为私有 + ] + ], +] +``` + +### 两个快捷方法 + +1. `move_local_file_to_filesystem($local_file_path, $save_file_path, $private = false, $bucket = 'aliyuncs', $update_when_exist = true)` + 将本地文件通过 `filesystem` 指定的介质来存储 +2. `filesystem_private_url($save_file_path, $timeout = 60, $bucket = 'aliyuncs')` + 获取私有文件的临时访问链接 + +### 提示 + +如果存储方式为 `本地`, 我们 增加了 `file.storage.local.cdn` 这个配置项, 用于生成可用的访问链接. + +如果使用的其他存储介质, 请记得安装相应扩展包. + +!> 当前仅对 `本地`, `阿里云oss` 做了适配, 因为没有其他元的账号..., 如果您当前使用的存储介质(如: 腾讯云), 请反馈给我们, 如能提供测试账号验证下更好. + diff --git a/docs/backend/functions.md b/docs/backend/functions.md index 832de4f..ab4b71d 100644 --- a/docs/backend/functions.md +++ b/docs/backend/functions.md @@ -52,9 +52,9 @@ #### register_route -#### move_local_file_to_oss +#### move_local_file_to_filesystem -#### oss_private_url +#### filesystem_private_url #### call_self_api diff --git a/src/admin/src/Controller/UploadController.php b/src/admin/src/Controller/UploadController.php index 74b2eda..c899a57 100644 --- a/src/admin/src/Controller/UploadController.php +++ b/src/admin/src/Controller/UploadController.php @@ -1,7 +1,6 @@ request->input('bucket', 'local'); + $private = $this->request->input('private', false); + $file = $this->request->file('file'); - if(!$file->isValid()) { + if (!$file->isValid()) { return $this->fail(ErrorCode::CODE_ERR_PARAM); } + $tmp_file = $file->toArray()['tmp_file']; $md5_filename = md5_file($tmp_file); $path = '1/' . date('Ym') . '/' . $md5_filename . '.' . $file->getExtension(); - $bucket = $this->request->input('bucket', 'aliyuncs'); - $private = $this->request->input('private', false); + try { - $uploaded = move_local_file_to_oss($tmp_file, $path, $private, $bucket); - if($uploaded === false) { + $uploaded = move_local_file_to_filesystem($tmp_file, $path, $private, $bucket); + if ($uploaded === false) { return $this->fail(ErrorCode::CODE_ERR_SERVER, '上传失败'); } [$width, $height] = getimagesize($tmp_file); @@ -35,7 +37,7 @@ class UploadController extends Controller ]; return $this->success($info); - } catch (OssException $e) { + } catch (\Exception $e) { Log::get('upload')->error($e->getMessage()); return $this->fail(ErrorCode::CODE_ERR_SERVER, $e->getMessage()); @@ -45,11 +47,13 @@ class UploadController extends Controller public function privateFileUrl() { $oss_path = $this->request->input('key'); - if(!$oss_path) { + $bucket = $this->request->input('storage', config('file.default')); + + if (!$oss_path) { return $this->fail(ErrorCode::CODE_ERR_PARAM); } - $private_url = oss_private_url($oss_path); - if(!$private_url) { + $private_url = filesystem_private_url($oss_path, MINUTE * 5, $bucket); + if (!$private_url) { return $this->fail(ErrorCode::CODE_ERR_SYSTEM); } diff --git a/src/admin/src/Service/ExportService.php b/src/admin/src/Service/ExportService.php index 69c52af..e62ef13 100644 --- a/src/admin/src/Service/ExportService.php +++ b/src/admin/src/Service/ExportService.php @@ -129,8 +129,8 @@ class ExportService ])->save(); $query['_page'] += 1; } - $bucket = config('storager.export_service', config('storager.default')); - $info = move_local_file_to_oss($file_path, '1/export_task/' . $file_name, true, $bucket); + $bucket = config('file.export_storage', config('file.default')); + $info = move_local_file_to_filesystem($file_path, '1/export_task/' . $file_name, true, $bucket); if($info) { $task->fill([ 'status' => ExportTasks::STATUS_SUCCESS, diff --git a/src/base-utils/composer.json b/src/base-utils/composer.json index 81ddc81..bf03382 100644 --- a/src/base-utils/composer.json +++ b/src/base-utils/composer.json @@ -60,6 +60,9 @@ "src/Helper/common.php", "src/Helper/array.php", "src/Helper/system.php" + ], + "classmap": [ + "src/classmap" ] }, "autoload-dev": { diff --git a/src/base-utils/src/Helper/system.php b/src/base-utils/src/Helper/system.php index 1081149..ae6cf57 100644 --- a/src/base-utils/src/Helper/system.php +++ b/src/base-utils/src/Helper/system.php @@ -1,23 +1,23 @@ $bucket]); + $filesystem = make(FilesystemFactory::class)->get($bucket); try { - $method = $private ? 'uploadPrivateFile' : 'uploadFile'; - $oss->$method($oss_file_path, $local_file_path); - $file_path = config('storager.aliyuncs.cdn') . '/' . $oss_file_path; - if($private) { - $file_path = oss_private_url($oss_file_path, MINUTE * 5, $bucket); + $stream = fopen($local_file_path, 'r'); + $has = $filesystem->has($save_file_path); + if (!$has) { + $filesystem->writeStream($save_file_path, $stream); + } + if ($has && $update_when_exist) { + $filesystem->updateStream($save_file_path, $stream); } + if ($private) { + $filesystem->setVisibility($save_file_path, AdapterInterface::VISIBILITY_PRIVATE); + } + $meta = $filesystem->getMetadata($save_file_path); + switch (config("file.storage.{$bucket}.driver")) { + case \Hyperf\Filesystem\Adapter\LocalAdapterFactory::class: + $file_path = config("file.storage.{$bucket}.cdn") . $meta['path']; + break; + case \Hyperf\Filesystem\Adapter\AliyunOssAdapterFactory::class; + $file_path = $meta['info']['url']; + break; + // TODO 更多 filesystem 渠道 + default: + $file_path = $save_file_path; + } + if ($private) { + $file_path = filesystem_private_url($save_file_path, MINUTE * 5, $bucket); + } return [ - 'file_path' => $file_path, - 'path' => 'oss/' . $oss_file_path, + 'file_path' => $file_path, // 直接可访问的完整链接 + 'path' => $save_file_path, // 资源的存储路径 ]; - } catch (OssException $exception) { - Log::get('move_local_file_to_oss')->error($exception->getMessage()); + } catch (\Exception $exception) { + Log::get('move_local_file_to_filesystem')->error($exception->getMessage()); return false; } } } -if(!function_exists('oss_private_url')) { - function oss_private_url($oss_file_path, $timeout = 60, $bucket = 'aliyuncs') +if (!function_exists('filesystem_private_url')) { + function filesystem_private_url($save_file_path, $timeout = 60, $bucket = 'aliyuncs') { - /** @var AliyunOSS $oss */ - $oss = make(AliyunOSS::class, ['bucket' => $bucket]); - $key = preg_replace('@^oss/@', '', $oss_file_path); + if (!$save_file_path) { + return false; + } + $filesystem = make(FilesystemFactory::class)->get($bucket); + if (Str::startsWith($save_file_path, 'http')) { + $save_file_path = parse_url($save_file_path)['path']; + $parts = explode('/', $save_file_path); + unset($parts[0]); + $save_file_path = implode('/', $parts); + } + $save_file_path = preg_replace('@^oss/@', '', $save_file_path); try { - return str_replace('-internal', '', $oss->getSignUrl($key, $timeout)); - } catch (OssException $exception) { - Log::get('oss_private_url')->error($exception->getMessage()); + switch (config("file.storage.{$bucket}.driver")) { + case \Hyperf\Filesystem\Adapter\LocalAdapterFactory::class: + return config("file.storage.{$bucket}.cdn") . $save_file_path; + break; + case \Hyperf\Filesystem\Adapter\AliyunOssAdapterFactory::class; + $adapter = $filesystem->getAdapter(); + if (method_exists($adapter, 'signUrl')) { + return $adapter->signUrl($save_file_path, $timeout); + } + break; + // TODO 更多 filesystem 渠道 + default: + return $save_file_path; + } + } catch (\Exception $exception) { + Log::get('filesystem_private_url')->error($exception->getMessage()); return false; } } } -if(!function_exists('call_self_api')) { +if (!function_exists('call_self_api')) { function call_self_api($api, $params = [], $method = 'GET') { $headers = [ @@ -121,12 +161,12 @@ if(!function_exists('call_self_api')) { } } -if(!function_exists('select_options')) { +if (!function_exists('select_options')) { function select_options($api, array $kws) { $ret = []; $chunk = array_chunk($kws, 100); - foreach($chunk as $part) { + foreach ($chunk as $part) { $ret = array_merge($ret, call_self_api($api, ['kw' => implode(',', $part)])); } @@ -134,17 +174,17 @@ if(!function_exists('select_options')) { } } -if(!function_exists('process_list_filter')) { +if (!function_exists('process_list_filter')) { function process_list_filter($processes, $rule) { - if(!$rule) { + if (!$rule) { return $processes; } - if($ignore = $rule['ignore'] ?? false) { - if(is_string($ignore) && $ignore === 'all') { + if ($ignore = $rule['ignore'] ?? false) { + if (is_string($ignore) && $ignore === 'all') { $processes = []; } - if(is_array($ignore)) { + if (is_array($ignore)) { $processes = array_filter($processes, function ($item) use ($ignore) { return !Str::startsWith($item, array_map(function ($each) { return Str::replaceLast('*', '', $each); @@ -152,7 +192,7 @@ if(!function_exists('process_list_filter')) { }); } } - if($active = $rule['active'] ?? []) { + if ($active = $rule['active'] ?? []) { $processes = array_merge($processes, $active); } @@ -160,17 +200,13 @@ if(!function_exists('process_list_filter')) { } } -if(!function_exists('get_sub_dir')) { +if (!function_exists('get_sub_dir')) { function get_sub_dir($dir, $exclude = []) { $paths = []; - $dirs = \Symfony\Component\Finder\Finder::create() - ->in($dir) - ->depth('<1') - ->exclude((array)$exclude) - ->directories(); + $dirs = \Symfony\Component\Finder\Finder::create()->in($dir)->depth('<1')->exclude((array)$exclude)->directories(); /** @var SplFileInfo $dir */ - foreach($dirs as $dir) { + foreach ($dirs as $dir) { $paths[] = $dir->getRealPath(); } @@ -178,7 +214,7 @@ if(!function_exists('get_sub_dir')) { } } -if(!function_exists('db_complete')) { +if (!function_exists('db_complete')) { function db_complete(array $conf) { return array_overlay($conf, [ diff --git a/src/base-utils/src/Middleware/HttpLogMiddleware.php b/src/base-utils/src/Middleware/HttpLogMiddleware.php index b0e6fa2..65bec3e 100644 --- a/src/base-utils/src/Middleware/HttpLogMiddleware.php +++ b/src/base-utils/src/Middleware/HttpLogMiddleware.php @@ -49,10 +49,14 @@ class HttpLogMiddleware implements MiddlewareInterface case Dispatcher::FOUND: if(!in_array($path, $uriEx)) { $response_content = mb_substr($response->getBody()->getContents(), 0, 125); + $request_content = $request->getBody()->getContents(); + if (preg_match('/filename=.(.*)/', $request_content, $m)) { + $request_content = urlencode($m[1]); + } $msg = [ 'referer' => $referer, 'uri' => $request->getRequestTarget(), - 'request' => $request->getBody()->getContents(), + 'request' => $request_content, 'use_time' => 1000 * (microtime(true) - $start_time), 'response' => $response_content, ]; diff --git a/src/base-utils/src/Scaffold/Controller/AbstractController.php b/src/base-utils/src/Scaffold/Controller/AbstractController.php index 373402e..c342511 100644 --- a/src/base-utils/src/Scaffold/Controller/AbstractController.php +++ b/src/base-utils/src/Scaffold/Controller/AbstractController.php @@ -265,7 +265,7 @@ abstract class AbstractController extends Controller if (is_callable($item['render'])) { $each[$item['field']] = $item['render']($each[$item['field']] ?? null, $each); } elseif (is_string($item['render']) && method_exists($this, $item['render'])) { - $each[$item['field']] = $this->$item['render']($each[$item['field']] ?? null, $each); + $each[$item['field']] = $this->{$item['render']}($each[$item['field']] ?? null, $each); } unset($each); } diff --git a/src/base-utils/src/classmap/OssAdapter.php b/src/base-utils/src/classmap/OssAdapter.php new file mode 100644 index 0000000..c9e322e --- /dev/null +++ b/src/base-utils/src/classmap/OssAdapter.php @@ -0,0 +1,451 @@ +supports = new Supports(); + try { + $this->bucket = $config['bucket']; + empty($config['endpoint']) ? null : $this->endpoint = $config['endpoint']; + empty($config['timeout']) ? $config['timeout'] = 3600 : null; + empty($config['connectTimeout']) ? $config['connectTimeout'] = 10 : null; + + if (!empty($config['isCName'])) { + $isCName = true; + } + if (!empty($config['token'])) { + $token = $config['token']; + } + $this->oss = new OssClient( + $config['accessId'], $config['accessSecret'], $this->endpoint, $isCName, $token + ); + $this->oss->setTimeout($config['timeout']); + $this->oss->setConnectTimeout($config['connectTimeout']); + } catch (Exception $e) { + throw $e; + } + } + + /** + * Write a new file. + * + * @param string $path + * @param string $contents + * @param Config $config Config object + * + * @return array|false false on failure file meta data on success + */ + public function write($path, $contents, Config $config) + { + $result = $this->oss->putObject($this->bucket, $path, $contents, $this->getOssOptions($config)); + $this->supports->setFlashData($result); + return $result; + } + + /** + * Write a new file using a stream. + * + * @param string $path + * @param resource $resource + * @param Config $config Config object + * + * @return array|false false on failure file meta data on success + */ + public function writeStream($path, $resource, Config $config) + { + if (!is_resource($resource)) { + return false; + } + $i = 0; + $bufferSize = 1000000; // 1M + while (!feof($resource)) { + if (false === $buffer = fread($resource, $block = $bufferSize)) { + return false; + } + $position = $i * $bufferSize; + $size = $this->oss->appendObject($this->bucket, $path, $buffer, $position, $this->getOssOptions($config)); + $i++; + } + fclose($resource); + return true; + } + + /** + * Update a file. + * + * @param string $path + * @param string $contents + * @param Config $config Config object + * + * @return array|false false on failure file meta data on success + */ + public function update($path, $contents, Config $config) + { + $result = $this->oss->putObject($this->bucket, $path, $contents, $this->getOssOptions($config)); + $this->supports->setFlashData($result); + return $result; + } + + /** + * Update a file using a stream. + * + * @param string $path + * @param resource $resource + * @param Config $config Config object + * + * @return array|false false on failure file meta data on success + */ + public function updateStream($path, $resource, Config $config) + { + $result = $this->write($path, stream_get_contents($resource), $config); + if (is_resource($resource)) { + fclose($resource); + } + return $result; + } + + /** + * Rename a file. + * + * @param string $path + * @param string $newpath + * + * @return bool + */ + public function rename($path, $newpath) + { + $this->oss->copyObject($this->bucket, $path, $this->bucket, $newpath); + $this->oss->deleteObject($this->bucket, $path); + return true; + } + + /** + * Copy a file. + * + * @param string $path + * @param string $newpath + * + * @return bool + */ + public function copy($path, $newpath) + { + $this->oss->copyObject($this->bucket, $path, $this->bucket, $newpath); + return true; + } + + /** + * Delete a file. + * + * @param string $path + * + * @return bool + */ + public function delete($path) + { + $this->oss->deleteObject($this->bucket, $path); + return true; + } + + /** + * Delete a directory. + * + * @param string $dirname + * + * @return bool + */ + public function deleteDir($dirname) + { + $lists = $this->listContents($dirname, true); + if (!$lists) { + return false; + } + $objectList = []; + foreach ($lists as $value) { + $objectList[] = $value['path']; + } + $this->oss->deleteObjects($this->bucket, $objectList); + return true; + } + + /** + * Create a directory. + * + * @param string $dirname directory name + * @param Config $config + * + * @return array|false + */ + public function createDir($dirname, Config $config) + { + $this->oss->createObjectDir($this->bucket, $dirname); + return true; + } + + /** + * Set the visibility for a file. + * + * @param string $path + * @param string $visibility + * + * @return array|false file meta data + * + * Aliyun OSS ACL value: 'default', 'private', 'public-read', 'public-read-write' + */ + public function setVisibility($path, $visibility) + { + $this->oss->putObjectAcl( + $this->bucket, + $path, + ($visibility == 'public') ? 'public-read' : 'private' + ); + return true; + } + + /** + * Check whether a file exists. + * + * @param string $path + * + * @return array|bool|null + */ + public function has($path) + { + return $this->oss->doesObjectExist($this->bucket, $path); + } + + /** + * Read a file. + * + * @param string $path + * + * @return array|false + */ + public function read($path) + { + return [ + 'contents' => $this->oss->getObject($this->bucket, $path) + ]; + } + + /** + * Read a file as a stream. + * + * @param string $path + * + * @return array|false + */ + public function readStream($path) + { + $resource = 'http://' . $this->bucket . '.' . $this->endpoint . '/' . $path; + return [ + 'stream' => $resource = fopen($resource, 'r') + ]; + } + + /** + * List contents of a directory. + * + * @param string $directory + * @param bool $recursive + * + * @return array + */ + public function listContents($directory = '', $recursive = false) + { + $directory = rtrim($directory, '\\/'); + + $result = []; + $nextMarker = ''; + while (true) { + // max-keys 用于限定此次返回object的最大数,如果不设定,默认为100,max-keys取值不能大于1000。 + // prefix 限定返回的object key必须以prefix作为前缀。注意使用prefix查询时,返回的key中仍会包含prefix。 + // delimiter是一个用于对Object名字进行分组的字符。所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组元素 + // marker 用户设定结果从marker之后按字母排序的第一个开始返回。 + $options = [ + 'max-keys' => 1000, + 'prefix' => $directory . '/', + 'delimiter' => '/', + 'marker' => $nextMarker, + ]; + $res = $this->oss->listObjects($this->bucket, $options); + + // 得到nextMarker,从上一次$res读到的最后一个文件的下一个文件开始继续获取文件列表 + $nextMarker = $res->getNextMarker(); + $prefixList = $res->getPrefixList(); // 目录列表 + $objectList = $res->getObjectList(); // 文件列表 + if ($prefixList) { + foreach ($prefixList as $value) { + $result[] = [ + 'type' => 'dir', + 'path' => $value->getPrefix() + ]; + if ($recursive) { + $result = array_merge($result, $this->listContents($value->getPrefix(), $recursive)); + } + } + } + if ($objectList) { + foreach ($objectList as $value) { + if (($value->getSize() === 0) && ($value->getKey() === $directory . '/')) { + continue; + } + $result[] = [ + 'type' => 'file', + 'path' => $value->getKey(), + 'timestamp' => strtotime($value->getLastModified()), + 'size' => $value->getSize() + ]; + } + } + if ($nextMarker === '') { + break; + } + } + + return $result; + } + + /** + * Get all the meta data of a file or directory. + * + * @param string $path + * + * @return array|false + */ + public function getMetadata($path) + { + return $this->oss->getObjectMeta($this->bucket, $path); + } + + /** + * Get the size of a file. + * + * @param string $path + * + * @return array|false + */ + public function getSize($path) + { + $response = $this->oss->getObjectMeta($this->bucket, $path); + return [ + 'size' => $response['content-length'] + ]; + } + + /** + * Get the mimetype of a file. + * + * @param string $path + * + * @return array|false + */ + public function getMimetype($path) + { + $response = $this->oss->getObjectMeta($this->bucket, $path); + return [ + 'mimetype' => $response['content-type'] + ]; + } + + /** + * Get the timestamp of a file. + * + * @param string $path + * + * @return array|false + */ + public function getTimestamp($path) + { + $response = $this->oss->getObjectMeta($this->bucket, $path); + return [ + 'timestamp' => $response['last-modified'] + ]; + } + + /** + * Get the visibility of a file. + * + * @param string $path + * + * @return array|false + */ + public function getVisibility($path) + { + $response = $this->oss->getObjectAcl($this->bucket, $path); + return [ + 'visibility' => $response, + ]; + } + + /** + * Get OSS Options + * @param Config $config + * @return array + */ + private function getOssOptions(Config $config) + { + $options = []; + if ($config->has("headers")) { + $options['headers'] = $config->get("headers"); + } + + if ($config->has("Content-Type")) { + $options["Content-Type"] = $config->get("Content-Type"); + } + + if ($config->has("Content-Md5")) { + $options["Content-Md5"] = $config->get("Content-Md5"); + $options["checkmd5"] = false; + } + return $options; + } + + public function signUrl($path, $timeout = 60) + { + return $this->oss->signUrl($this->bucket, $path, $timeout); + } + +} diff --git a/src/cron-center/src/ClassJobAbstract.php b/src/cron-center/src/ClassJobAbstract.php index c28845f..2152fbb 100644 --- a/src/cron-center/src/ClassJobAbstract.php +++ b/src/cron-center/src/ClassJobAbstract.php @@ -18,26 +18,24 @@ abstract class ClassJobAbstract $this->job = $crontab; $this->jobManager = make(CronManager::class); $this->state = $this->jobManager->getJobState($crontab->getId()); - $this->logger = logger(); + $this->logger = logger()->get('cron_center'); } public function run($params = []) { - $func = function () use ($params) { - try { - $this->beforeAction($params); - $this->handle($params); - $this->afterAction($params); - } catch (\Throwable $throwable) { - $this->onError($throwable); - } finally { - $this->onComplete(); + try { + $this->beforeAction($params); + $result = $this->handle($params); + $this->afterAction($params); + return $result; + } catch (\Throwable $throwable) { + $this->onError($throwable); + return false; + } finally { + $this->onComplete(); - return $this->evaluate(); - } - }; - - return $func(); + return $this->evaluate(); + } } public function beforeAction($params = [])