From dae908b806c976410aa0795ad4369cb421b715d9 Mon Sep 17 00:00:00 2001 From: jsasg <735273025@qq.com> Date: Thu, 8 May 2025 18:19:46 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/command/DataMigration.php | 488 ++++++++++++++++++++++++++++++---- 1 file changed, 431 insertions(+), 57 deletions(-) diff --git a/app/command/DataMigration.php b/app/command/DataMigration.php index 61d9918f..1b48d496 100644 --- a/app/command/DataMigration.php +++ b/app/command/DataMigration.php @@ -26,9 +26,12 @@ class DataMigration extends Command protected function execute(Input $input, Output $output) { + ini_set('pcre.backtrack_limit', 10000000); + ini_set('default_socket_timeout', 1); + // 指令输出 $this->println = function ($msg) use ($output) { - $output->writeln($msg); + $output->writeln($msg); }; try { @@ -38,15 +41,30 @@ class DataMigration extends Command // 迁移产品分类 // $this->productCategory(); + // 迁移产品属性 + // $this->migrateProductAttr(); + + // 迁移产品 + $this->migrateProduct(); + // 迁移文章 // $this->migrateArticle([ - // 1 => 2, - // 2 => 3, - // 33 => 4, - // 36 => 5, - // 16 => 7, - // 31 => 8, - // 32 => 9 + // 68 => 10, + // 69 => 11, + // 70 => 12, + // 71 => 13, + // 72 => 14, + // 73 => 15, + // 74 => 16, + // 75 => 17, + // 78 => 19, + // 79 => 20, + // 80 => 21, + // 81 => 22, + // 82 => 23, + // 83 => 24, + // 84 => 25, + // 85 => 26 // ]); // 迁移faq @@ -72,29 +90,31 @@ class DataMigration extends Command // $this->migrateVideoCategory(); // 迁移视频 - $this->migrateVideo([ - 0 => 0, - 1 => 1, - 2 => 2, - 3 => 3, - 4 => 4, - 5 => 5, - 6 => 6, - 7 => 7, - 8 => 8, - 11 => 0, - 36 => 9, - 37 => 10, - 38 => 11, - 39 => 12, - 40 => 13, - 41 => 14, - 42 => 15, - 43 => 16, - 60 => 17, - 62 => 18, - 63 => 19 - ]); + // $this->migrateVideo([ + // 0 => 0, + // 1 => 1, + // 2 => 2, + // 3 => 3, + // 4 => 4, + // 5 => 5, + // 6 => 6, + // 7 => 7, + // 8 => 8, + // 11 => 0, + // 36 => 9, + // 37 => 10, + // 38 => 11, + // 39 => 12, + // 40 => 13, + // 41 => 14, + // 42 => 15, + // 43 => 16, + // 60 => 17, + // 62 => 18, + // 63 => 19 + // ]); + + // $this->test(); $output->writeln('success'); } catch(\Throwable $th) { @@ -174,6 +194,319 @@ class DataMigration extends Command } } + // 迁移产品属性 + private function migrateProductAttr() + { + $products = Db::connect('old') + ->name('product') + ->where('country_code', 'in', ['ZH', 'US']) + ->where('product_attr', '<>', '') + ->order(['id' => 'asc']) + ->cursor(); + + $exists = []; + foreach ($products as $v) + { + $attrs = []; + $product_attr = json_decode($v['product_attr'], true); + foreach ($product_attr as $attr) { + if (!in_array($attr, ['颜色', 'Color']) && !in_array($attr, $exists)) { + $attrs[] = [ + 'language_id' => $v['country_code'] == 'ZH' ? 1 : 2, + 'attr_type' => 2, + 'attr_name' => $attr, + 'is_system' => 0 + ]; + $exists[] = $attr; + } + } + if (empty($attrs)) { + continue; + } + Db::name('product_attr')->insertAll($attrs); + } + } + + // 迁移产品 + private function migrateProduct() + { + $attrs_map = []; + $attrs_dict = Db::name('product_attr') + ->withoutField(['created_at', 'updated_at', 'deleted_at']) + ->select(); + foreach ($attrs_dict as $attr) + { + $code = $attr['language_id'] == 1 ? 'ZH' : 'US'; + if (!isset($attrs_map[$code])) { + $attrs_map[$code] = []; + } + $attrs_map[$code][$attr['attr_name']] = $attr['id']; + } + + $old_db = Db::connect('old'); + + $products = $old_db->name('product') + ->where('id', '=', 128) + ->where('country_code', 'in', ['ZH', 'US']) + ->order(['id' => 'asc']) + ->cursor(); + + $uploadMgr = new UploadMannager(); + foreach ($products as $v) + { + Db::startTrans(); + try { + // 处理封面图片 + $image = ''; + $image_ret = $uploadMgr->upload($uploadMgr->download($v['list_bk_img']), 'image', 'product'); + if ($image_ret['code'] == 0) { + $image = $image_ret['data']['path']; + } else { + $image = $image_ret['msg']; + } + + // 处理视频 + $video = ''; + $video_ret = $uploadMgr->upload($uploadMgr->download($v['videopath']), 'video', 'video'); + if ($video_ret['code'] == 0) { + $video = $video_ret['data']['path']; + } else { + $video = $video_ret['msg']; + } + + // 处理详情中图片 + $content = preg_replace_callback('/]*src=[\'"]([^\'"]*?)[\'"][^>]*>/is', function($matches) use ($uploadMgr) { + $file_path = ''; + try { + $ret = $uploadMgr->upload($uploadMgr->download($matches[1]), 'image'); + if ($ret['code'] == 0) { + $file_path = $ret['data']['path']; + } + } catch (\Throwable $th) { + $this->println($th->getMessage() . ':' . $th->getLine()); + return ''; + } + return ''; + }, $v['ld_md_content']); + + $item = [ + 'language_id' => $v['country_code'] == 'ZH' ? 1 : 2, + 'category_id' => '', + 'spu' => $v['brand_id'], + 'name' => $v['name'], + 'short_name' => $v['shortname'], + 'cover_image' => $image, + 'desc' => $v['description'], + 'video_img' => '', + 'video_url' => $video, + 'is_sale' => $v['is_onsale'], + 'is_new' => $v['isnew'], + 'is_hot' => $v['ishot'], + 'is_show' => $v['is_show'], + 'sort' => $v['sort'], + 'detail' => $content, + 'status' => $v['stat'] == -1 ? -1 : 1, + 'seo_title' => $v['seo_title'], + 'seo_keywords' => $v['seo_keyword'], + 'seo_desc' => $v['seo_description'], + 'created_at' => date('Y-m-d H:i:s', $v['createtime']), + 'updated_at' => date('Y-m-d H:i:s', $v['updatetime']), + 'deleted_at' => $v['stat'] == -1 ? date('Y-m-d H:i:s') : null, + ]; + // 保存产品数据 + $id = Db::name('product')->insertGetId($item); + + // 保存产品参数数据 + $prarms = []; + $views = unserialize($v['product_view']); + foreach ($views as $p) { + $prarms[] = [ + 'product_id' => $id, + 'name' => $p['desc_title'], + 'value' => $p['desc_desc'] + ]; + } + Db::name('product_params')->insertAll($prarms); + + // 保存sku数据 + $skus = []; + $sku_images = $old_db->name('product_image') + ->where('product_id', '=', $v['id']) + ->where('country_code', '=', $v['country_code']) + ->where('stat', '=', 0) + ->order(['id' => 'asc']) + ->select(); + $images = []; + $photo_album = []; + foreach ($sku_images as $im) { + $pkey = $im['sku']; + if (empty($pkey)) { + if (!empty($im['image_color'])) { + $pkey = md5($im['image_color']); + } else { + $pkey = 0; + } + } + if (!isset($images[$pkey])) { + $images[$pkey] = []; + $images[$pkey]['sku'] = $im['sku']; + } + + // 处理图册 + if (!empty($im['image_url'])) { + $photos = json_decode($im['image_url'], true); + if (empty($photos)) { + $photos[] = $im['image_url']; + } + foreach ($photos as $photo) { + $photos_ret = $uploadMgr->upload($uploadMgr->download($photo), 'image', 'product'); + if ($photos_ret['code'] == 0) { + if (!isset($photo_album[$pkey])) { + $photo_album[$pkey] = []; + } + $photo_album[$pkey][] = $photos_ret['data']['path']; + } + } + } + + // 处理属性 + $attrs = json_decode($im['image_color'], true); + if (!empty($attrs)) { + foreach ($attrs as $k => $at) { + if ($k != 'sort') { + $attr_value = $at; + if (in_array($k, ['颜色', 'Color'])) { + if ($k == 'Color') $k = '颜色'; + $attr_value = '/static/common/images/colors/' . $at . '.png'; + } + $images[$pkey]['attrs'] = [ + 'attr_id' => $attrs_map[$v['country_code']][$k], + 'attr_value' => $attr_value, + ]; + } + } + } else { + $attr_value = $im['image_color']; + if (empty($attr_value) || $attr_value == '[]') { + $images[$pkey]['attrs'] = []; + continue; + } + if (\think\helper\Str::endsWith($attr_value, ['.png', '.jpg', 'jpeg', '.gif'])) { + $photos_ret = $uploadMgr->upload($uploadMgr->download($attr_value), 'image', 'product'); + if ($photos_ret['code'] == 0) { + $attr_value = $photos_ret['data']['path']; + } + } + $attr_arr = [ + 'attr_id' => $attrs_map[$v['country_code']]['颜色'], + 'attr_value' => $attr_value, + ]; + if ( + empty($images[$pkey]['attrs']) || + (!empty($images[$pkey]['attrs']) && empty(array_intersect($images[$pkey]['attrs'], $attr_arr))) + ) { + + $images[$pkey]['attrs'][] = $attr_arr; + file_put_contents(runtime_path() . 'attrs.txt', json_encode($attr_arr)); + } + } + } + foreach ($images as $key => $image) { + $images[$key]['photo_album'] = $photo_album[$key]; + } + file_put_contents(runtime_path() . 'images.txt', json_encode($images, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)); + file_put_contents(runtime_path() . 'photo_album.txt', json_encode($photo_album, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)); + + $two_images = $old_db->name('product_two_img') + ->where('product_id', '=', $v['id']) + ->where('country_code', '=', $v['country_code']) + ->where('stat', '=', 0) + ->order(['id' => 'asc']) + ->select(); + foreach ($two_images as $ti) { + $tpkey = 0; + if (!empty($ti['image_color'])) { + $tpkey = md5($ti['image_color']); + } + if (!empty($ti['image_url'])) { + $main_image_ret = $uploadMgr->upload($uploadMgr->download($ti['image_url']), 'image', 'product'); + if ($main_image_ret['code'] == 0) { + $ti['image_url'] = $main_image_ret['data']['path']; + } + } + $skus[] = ['main_image' => $ti['image_url'], 'image_color' => $ti['image_color'], 'pkey' => $tpkey]; + } + if (!empty($skus)) { + $temp = []; + foreach ($skus as $idx => $sku) { + if (isset($images[$sku['pkey']])) { + $skus[$idx]['product_id'] = $id; + $skus[$idx]['sku'] = $images[$sku['pkey']]['sku']; + $skus[$idx]['photo_album'] = $images[$sku['pkey']]['photo_album']; + $skus[$idx]['attrs'] = $images[$sku['pkey']]['attrs']; + } else { + if (!empty($images[$idx])) { + $temp = $images[$idx]; + } + if (!empty($temp)) { + $skus[$idx]['product_id'] = $id; + $skus[$idx]['sku'] = $temp['sku']; + $skus[$idx]['photo_album'] = $temp['photo_album']; + $skus[$idx]['attrs'] = $temp['attrs']; + } + } + unset($skus[$idx]['pkey']); + unset($skus[$idx]['image_color']); + } + } else { + foreach ($images as $image) { + $skus[] = [ + 'product_id' => $id, + 'sku' => $image['sku'], + 'main_image' => '', + 'photo_album' => $image['photo_album'], + 'skus' => $image['attrs'] + ]; + } + } + file_put_contents(runtime_path() . 'skus.txt', json_encode($skus, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)); + // throw new \Exception("exit;"); + + foreach ($skus as $sku) { + $sku_id = Db::name('product_sku')->insertGetId([ + 'product_id' => $sku['product_id'], + 'sku' => $sku['sku'], + 'main_image' => $sku['main_image'], + 'photo_album' => json_encode($sku['photo_album']), + ]); + if (!empty($sku['attrs'])) { + foreach ($sku['attrs'] as $attr) { + if (!empty($sku_id)) { + Db::name('product_sku_attr')->insert([ + 'sku_id' => $sku_id, + 'attr_id' => $attr['attr_id'], + 'attr_value' => $attr['attr_value'] + ]); + } + } + } + } + Db::commit(); + $this->println(sprintf('迁移产品ID：%s => %s', $v['id'], $id)); + } catch (\Throwable $th) { + Db::rollback(); + file_put_contents( + runtime_path() . 'product_throwable.txt', + sprintf('【%s】 产品【%s】，迁移失败，错误【%s:%d】' . PHP_EOL, date('Y-h-d H:i:s'), $v['id'], $th->getMessage(), $th->getLine()) + ); + file_put_contents( + runtime_path() . 'product_throwable_details.txt', + (string)$th + ); + } + } + } + // 迁移文章 private function migrateArticle($category_map = []) { @@ -201,22 +534,19 @@ class DataMigration extends Command } // 处理详情中图片 - $content = $v['content']; - preg_match_all('//i', $content, $matches); - $content_images = []; - foreach ($matches[1] as $val) { + $content = preg_replace_callback('/]*src=[\'"]([^\'"]*?)[\'"][^>]*>/is', function ($matches) use ($uploadMgr) { + $file_path = ''; try { - $ret = $uploadMgr->upload($uploadMgr->download($val), 'image'); + $ret = $uploadMgr->upload($uploadMgr->download($matches[1]), 'image'); if ($ret['code'] == 0) { - $content_images[$val] = $ret['data']['path']; + $file_path = $ret['data']['path']; } } catch (\Throwable $th) { - continue; + $this->println($th->getMessage() . ':' . $th->getLine()); + return ''; } - } - foreach ($content_images as $key => $val) { - $content = str_replace($key, $val, $content); - } + return ''; + }, $v['content']); $item = [ 'language_id' => $v['country_code'] == 'ZH' ? 1 : 2, 'category_id' => $category_map[$v['cid']], @@ -234,12 +564,12 @@ class DataMigration extends Command 'seo_title' => $v['seo_title'], 'seo_keywords' => $v['seo_keyword'], 'seo_desc' => $v['seo_description'], - 'enabled' => $v['stat'], - 'release_time' => date('Y-m-d H:i:s', $v['createtime']) + 'release_time' => date('Y-m-d H:i:s', $v['createtime']), + 'deleted_at' => $v['stat'] == -1 ? date('Y-m-d H:i:s') : null ]; - Db::name('article')->insert($item); + $id = Db::name('article')->insertGetId($item); - $this->println('迁移文章ID：' . $v['id']); + $this->println(sprintf('迁移文章ID：%s => %s', $v['id'], $id)); } } @@ -422,7 +752,7 @@ class DataMigration extends Command class UploadMannager { const UPLOAD_BASE_API = 'http://dev.ow.f2b211.com'; - const DOWNLOAD_BASE_API = 'https://www.orico.com.cn'; + const DOWNLOAD_BASE_API = 'http://www.orico.com.cn'; const DOWNLOAD_TEMP_PATH = '/var/www/html/orico-official-website/public/migrate_temp_images'; private $username = 'admin'; private $password = 'Aa-1221'; @@ -453,20 +783,44 @@ class UploadMannager $url = self::DOWNLOAD_BASE_API . \think\helper\Str::substr($file_name, mb_strpos($file_name, $need) + mb_strlen($need)); } $file_name = '/' . \think\helper\Str::substr($url, mb_strpos($url, '://') + 3); - } else { + } + elseif (\think\helper\Str::startsWith($file_name, 'data:image/')) { + $idx = strpos($file_name, ';'); + $file_type = substr($file_name, 0, $idx); + $base64_image = preg_replace('#^data:image/\w+;base64,#i', '', $file_name); + $file_data = base64_decode($base64_image, true); + $file_path = self::DOWNLOAD_TEMP_PATH . '/uploads/' . date('Ymd') . '/' . uniqid() . '.' . substr($file_type, 11); + $dir = dirname($file_path); + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + if (!file_put_contents($file_path, $file_data, FILE_USE_INCLUDE_PATH)) { + throw new \Exception('转换base64图片失败'); + } + return $file_path; + } + else { $url = self::DOWNLOAD_BASE_API . str_replace(" ", "%20", $file_name); } $file_path = self::DOWNLOAD_TEMP_PATH . $file_name; - $opts = [ - 'http' => [ - 'method' => 'GET', - 'header' => 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64)' - ] - ]; - $context = stream_context_create($opts); - $file = file_get_contents($url, false, $context); - $dir = dirname($file_path); + if (file_exists($file_path)) { + return $file_path; + } + + // 使用file_get_contents下载 + // $opts = [ + // 'http' => [ + // 'method' => 'GET', + // 'header' => 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64)' + // ] + // ]; + // $context = stream_context_create($opts); + // $file = file_get_contents($url, false, $context); + + // 使用curl下载 + $file = $this->file_get_withcurl($url); + $dir = dirname($file_path); if (!is_dir($dir)) { mkdir($dir, 0777, true); } @@ -474,6 +828,26 @@ class UploadMannager return $file_path; } + // 使用curl下载图片 + private function file_get_withcurl($url) + { + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); // 连接超时 10 秒 + curl_setopt($ch, CURLOPT_TIMEOUT, 30); // 传输超时 30 秒 + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // 跟随重定向 + curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0'); // 模拟浏览器 UA + + $data = curl_exec($ch); + if ($data === false) { + error_log(sprintf('cURL Error: %s; URL: %s', curl_error($ch), $url)); + return false; + } + curl_close($ch); + + return $data; + } + // 上传图片 public function upload($file_path, $field_name, $module = 'unknown') { if (empty($file_path)) {