Fixes/gcs fixes (#2438)
* Ensure only files are uploaded since the node lib does not support uloading folders directly * Addig flag for enabeling gzip metadata encoding and disabled resumable upload since to many connecteins are opened some times. * Running the linter to late as usual
This commit is contained in:
parent
247074e777
commit
6b3209e4b3
|
|
@ -917,6 +917,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
|
|||
type: 'boolean',
|
||||
group: 'GoogleCloudStorage'
|
||||
})
|
||||
.option('gcs.gzip', {
|
||||
describe:
|
||||
'Add content-encoding for gzip to the uploaded files. Read more at https://cloud.google.com/storage/docs/transcoding. If you host your results directly from the bucket, gzip must be set to false',
|
||||
default: false,
|
||||
type: 'boolean',
|
||||
group: 'GoogleCloudStorage'
|
||||
})
|
||||
.option('gcs.path', {
|
||||
describe:
|
||||
"Override the default folder path in the bucket where the results are uploaded. By default it's " +
|
||||
|
|
|
|||
|
|
@ -21,7 +21,13 @@ async function upload(dir, gcsOptions, prefix) {
|
|||
const bucket = storage.bucket(gcsOptions.bucketname);
|
||||
|
||||
for (let file of files) {
|
||||
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
|
||||
const stats = fs.statSync(file);
|
||||
|
||||
if (stats.isFile()) {
|
||||
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
|
||||
} else {
|
||||
log.debug(`Will not upload ${file} since it is not a file`);
|
||||
}
|
||||
}
|
||||
return Promise.all(promises);
|
||||
}
|
||||
|
|
@ -33,9 +39,9 @@ async function uploadFile(file, bucket, gcsOptions, prefix, baseDir) {
|
|||
const params = {
|
||||
public: !!gcsOptions.public,
|
||||
destination: fileName,
|
||||
resumable: true,
|
||||
resumable: false,
|
||||
validation: 'crc32c',
|
||||
gzip: true,
|
||||
gzip: !!gcsOptions.gzip,
|
||||
metadata: {
|
||||
metadata: {
|
||||
cacheControl: 'public, max-age=31536000'
|
||||
|
|
@ -83,6 +89,10 @@ module.exports = {
|
|||
if (gcsOptions.removeLocalResult) {
|
||||
await fs.remove(baseDir);
|
||||
log.debug(`Removed local files and directory ${baseDir}`);
|
||||
} else {
|
||||
log.debug(
|
||||
`Local result files and directories are stored in ${baseDir}`
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
log.error('Could not upload to Google Cloud Storage', e);
|
||||
|
|
|
|||
Loading…
Reference in New Issue