Skip to content

Commit f7c7284

Browse files
New asset scraper (#1308)
1 parent 0deb682 commit f7c7284

File tree

23 files changed

+2881
-34
lines changed

23 files changed

+2881
-34
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,4 @@ tmp/
7979
# TypeScript
8080
packages/**/build/
8181
tsconfig.tsbuildinfo
82+
packages/mg-assetscraper-db/assets.sqlite

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"private": true,
3+
"version": "0.0.0",
34
"repository": "https://github.com/TryGhost/migrate",
45
"author": "Ghost Foundation",
56
"license": "MIT",
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
module.exports = {
2+
parser: '@typescript-eslint/parser',
3+
plugins: ['ghost'],
4+
extends: [
5+
'plugin:ghost/node'
6+
],
7+
rules: {
8+
'no-unused-vars': 'off', // doesn't work with typescript
9+
'no-undef': 'off', // doesn't work with typescript
10+
'ghost/ghost-custom/no-native-errors': 'off',
11+
'ghost/ghost-custom/no-native-error': 'off',
12+
'ghost/ghost-custom/ghost-error-usage': 'off',
13+
// todo: clean this up
14+
'ghost/filenames/match-regex': 'off'
15+
}
16+
};
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/build
2+
/tsconfig.tsbuildinfo
3+
/tmp
4+
.env
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2013-2023 Ghost Foundation
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
## WIP Asset Scraper
2+
3+
Readme coming soon…
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/** @type {import('ts-jest').JestConfigWithTsJest} */
2+
module.exports = {
3+
testEnvironment: 'node',
4+
transform: {},
5+
testTimeout: 60 * 1000
6+
};
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
{
2+
"name": "@tryghost/mg-assetscraper-db",
3+
"version": "0.1.0",
4+
"repository": "https://github.com/TryGhost/migrate/tree/main/packages/mg-assetscraper-db",
5+
"author": "Ghost Foundation",
6+
"license": "MIT",
7+
"type": "module",
8+
"main": "build/index.js",
9+
"types": "build/types.d.ts",
10+
"scripts": {
11+
"dev": "echo \"Implement me!\"",
12+
"build:watch": "tsc --watch --preserveWatchOutput --sourceMap",
13+
"build": "rm -rf build && rm -rf tsconfig.tsbuildinfo && tsc --build --sourceMap",
14+
"prepare": "yarn build",
15+
"lint": "eslint src/ --ext .ts --cache",
16+
"posttest": "yarn lint",
17+
"test": "rm -rf build && yarn build --force && NODE_OPTIONS=--experimental-vm-modules c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura jest build/test --runInBand"
18+
},
19+
"files": [
20+
"build"
21+
],
22+
"publishConfig": {
23+
"access": "public"
24+
},
25+
"devDependencies": {
26+
"@tryghost/mg-fs-utils": "0.15.1",
27+
"@types/jest": "^29.5.3",
28+
"@types/sinon": "^17.0.0",
29+
"@typescript-eslint/eslint-plugin": "^8.0.0",
30+
"@typescript-eslint/parser": "^8.0.0",
31+
"c8": "10.1.2",
32+
"dotenv": "16.4.5",
33+
"eslint": "8.57.0",
34+
"jest": "29.7.0",
35+
"nock": "13.5.5",
36+
"sinon": "^18.0.0",
37+
"typescript": "5.5.4"
38+
},
39+
"dependencies": {
40+
"@tryghost/debug": "0.1.32",
41+
"@tryghost/errors": "1.3.5",
42+
"@tryghost/listr-smart-renderer": "0.5.15",
43+
"@tryghost/mg-fs-utils": "0.12.14",
44+
"@tryghost/request": "1.0.9",
45+
"@tryghost/string": "0.2.12",
46+
"asakadb": "1.0.1",
47+
"cheerio": "1.0.0-rc.12",
48+
"file-type": "20.4.0",
49+
"fs-extra": "11.2.0",
50+
"heic-convert": "2.1.0",
51+
"sequelize": "6.37.5",
52+
"sharp": "0.33.5",
53+
"sqlite3": "5.1.7",
54+
"transliteration": "2.3.5",
55+
"uuid": "11.0.2"
56+
}
57+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import AssetScraper from './lib/AssetScraper.js';
2+
3+
export default AssetScraper;
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import {join} from 'node:path';
2+
import {Sequelize, DataTypes} from 'sequelize';
3+
4+
export default class AssetCache {
5+
// db: any;
6+
sequelize: any;
7+
Asset: any;
8+
assets: any;
9+
fileCache: any;
10+
11+
constructor({fileCache}: {fileCache: any}) {
12+
this.fileCache = fileCache;
13+
14+
const theCachePath = join(this.fileCache.tmpDir, 'assets-cache');
15+
16+
// Create the cache DB if it doesn't exist
17+
// if (!this.fileCache.hasFile(join('assets-cache', 'assets.db'), 'tmp')) {
18+
// this.fileCache.writeTmpFileSync('', join('assets-cache', 'assets.db'), false);
19+
// } else {
20+
// // console.log(`File '${theCachePath + '/assets.db'}' already exists.`);
21+
// }
22+
// if (!this.fileCache.hasFile(join('assets', 'assets.db'), 'tmp')) {
23+
// this.fileCache.writeTmpFileSync('', join('assets', 'assets.db'), false);
24+
// } else {
25+
// // console.log(`File '${theCachePath + '/assets.db'}' already exists.`);
26+
// }
27+
28+
// const sequelize = new Sequelize('sqlite::memory:');
29+
// const sequelize = new Sequelize(theCachePath + '/assets.db');
30+
const sequelize = new Sequelize({
31+
dialect: 'sqlite',
32+
storage: theCachePath + '/assets.db',
33+
logging: false
34+
});
35+
const Asset = sequelize.define('Asset', {
36+
src: DataTypes.STRING,
37+
status: DataTypes.NUMBER,
38+
localPath: DataTypes.STRING
39+
});
40+
41+
this.sequelize = sequelize;
42+
this.Asset = Asset;
43+
// this.db = db;
44+
}
45+
46+
async init() {
47+
await this.sequelize.sync();
48+
}
49+
50+
/**
51+
* Add an item to the asset cache list
52+
* TODO: findOrCreate might make sense here
53+
*/
54+
async add(src: string) {
55+
// Check if it exists first, and return that if so
56+
const existingAsset = await this.Asset.findOne({where: {src: src}});
57+
58+
if (existingAsset) {
59+
return existingAsset;
60+
}
61+
62+
// If not, insert it and return the new item
63+
const newAsset = await this.Asset.create({
64+
src: src
65+
});
66+
67+
return newAsset;
68+
}
69+
70+
async update(id: string, key: any, value: any) {
71+
// this.db.update((item: any) => item.uuid === uuid, (item: any) => ({...item, [key]: value}));
72+
73+
return this.Asset.update({
74+
[key]: value
75+
},
76+
{
77+
where: {
78+
id: id
79+
}
80+
});
81+
}
82+
83+
/**
84+
* Get all the items from the asset cache list
85+
*/
86+
async getAll() {
87+
// TODO: Add pagination
88+
// return this.db.get();
89+
return this.Asset.findAll();
90+
}
91+
92+
/**
93+
* Find an individual item form the asset cache list
94+
*/
95+
// async find(clauses: any) {
96+
// return this.db.get((item: any) => clauses(item));
97+
// }
98+
99+
/**
100+
* A common method to find an item by its src
101+
*/
102+
async findBySrc(src: string) {
103+
// return this.db.getOne((item: any) => item.src === src); // Returns the record with id 1
104+
const existingAsset = await this.Asset.findOne({where: {src: src}});
105+
return existingAsset;
106+
}
107+
108+
/**
109+
* Delete an item from the asset cache list
110+
*/
111+
// async deleteBySrc(src: string) {
112+
// this.db.delete((item: any) => item.src === src);
113+
// }
114+
115+
/**
116+
* Used in tests, not intended for general use
117+
*/
118+
async _reset() {
119+
/* c8 ignore next 3 */
120+
if (process.env.NODE_ENV !== 'test') {
121+
throw new Error('This method is only for use in tests');
122+
}
123+
124+
await this.Asset.drop();
125+
}
126+
}

0 commit comments

Comments
 (0)