@Alan
Here is the code (maybe this helps in analyse)
const taskLib = require('/lib/xp/task');
const feedUtils = require('/lib/feedUtils');
const httpClient = require("/lib/http-client");
const contentLib = require('/lib/xp/content');
const contextLib = require('/lib/xp/context');
exports.run = function (params) {
taskLib.progress({info: 'Task started'});
log.info("Start task with the following params %s", params);
const NEWS_API_URL = params.api_url;
const NEWS_FOLDER = params.news_folder;
const ROOT_PATH = params.root_path;
const REPOSITORY = params.repository;
if (!NEWS_API_URL) {
log.warning("You must specify an url in %S", app.name + '.cfg');
return;
}
// fetch the news from the given url
let newsResponse;
try {
newsResponse = httpClient.request({
url: NEWS_API_URL
})
} catch (e) {
log.error("Failed to fetch news from %s with the following message: %s", NEWS_API_URL, e.info);
taskLib.progress({info: 'Task aborted'});
return;
}
// exit function, when status not 200
if (newsResponse.status !== 200) {
log.warning('url %s responded with status %s', NEWS_API_URL, newsResponse.status);
taskLib.progress({info: 'Task aborted'});
return;
}
const newsItems = JSON.parse(newsResponse.body).items;
let newsCount;
newsItems ? newsCount = newsItems.length : newsCount = 0;
log.info(`Es wurden ${newsCount} Nachrichten gelesen`);
const newsFolderExists = runInContext(
() => checkFolderExists(ROOT_PATH + '/' + NEWS_FOLDER));
if (!newsFolderExists) {
log.info("Create Market News Folder")
//create folder
runInContext(
() => createDirectoryInRepo(
ROOT_PATH,
NEWS_FOLDER,
NEWS_FOLDER
),
REPOSITORY
)
}
let folderCreated = 0;
let newsCreated = 0;
newsItems.map((item, i) => {
const content = newsItems[i].content;
// extract image from content
// mostly higher resolution that what is in image
const imgSrc = feedUtils.extractImg(content);
// check image src and override image
if (imgSrc) {
newsItems[i].image = imgSrc;
}
//ensure that image url is encrypted
//to prevent browser warnings
newsItems[i].image = newsItems[i].image.replace("http:", "https:");
//remove image, anchors and line-breaks
newsItems[i].content = feedUtils.cleanContent(content);
//convert date
const date = new Date(newsItems[i].pubDate);
newsItems[i].pubDate = date.toISOString();
//get year and month in 2-digit format from date
const year = date.getFullYear().toString();
const month = ("0" + (date.getMonth() + 1)).slice(-2);
//prepare folders for news
//follows site/news/year/month
folderCreated += runInContext(
() => createDirectoryInRepo(ROOT_PATH + '/' + NEWS_FOLDER, year, year),
REPOSITORY
);
folderCreated += runInContext(
() => createDirectoryInRepo(`${ROOT_PATH}/${NEWS_FOLDER}/${year}`, month, `${month}/${year}`),
REPOSITORY
);
//finally, save the news-article in repo
newsCreated += runInContext(
() => saveToRepo(item, NEWS_FOLDER, ROOT_PATH),
REPOSITORY
);
});
log.info("NEWS-FETCHER: Created %s folders and %s news", folderCreated, newsCreated);
taskLib.progress({info: 'Task completed'});
}
/***
* creates sub-folders in repository
* @param baseDir path to create the folder in
* @param dirName name of the folder to create
* @param displayName name to show in UI
* @param templateId id of the page-template
*/
function createDirectoryInRepo(baseDir, dirName, displayName) {
try {
const folder = contentLib.create({
name: dirName,
displayName: displayName,
contentType: `${app.name}:marketNewsFolder`,
parentPath: baseDir,
data: "",
})
log.info("+++ created a new folder: %s +++", baseDir + '/' + dirName);
return 1;
} catch
(e) {
if (e.code === 'contentAlreadyExists') {
return 0;
} else {
log.error('Unexpected error: (%s) %s ', e.code, e.message);
return 0;
}
}
}
function addImageNode(node) {
const imgSrc = node.data.image;
if (!imgSrc) {
log.warning('News-Article %s does not contain an image', node._displayName);
return;
}
const imageName = getFileName(imgSrc);
//log.info("Fetching image...")
const resp = httpClient.request({
url: imgSrc
});
const cTypes = resp.contentType.split(';');
//let imageNode;
let imageAttachment
if (resp.status === 200) {
imageAttachment = contentLib.addAttachment({
key: node._path,
name: imageName ? imageName : "Image",
mimeType: cTypes[0],
//label: 'news-image',
data: resp.bodyStream
})
}
}
function saveToRepo(item, root, site_path) {
try {
const name = 'market_news_' + item.guid.contents;
const date = new Date(item.pubDate);
const year = date.getFullYear().toString();
const month = ("0" + (date.getMonth() + 1)).slice(-2);
const newMarketNews = contentLib.create({
name: name,
displayName: item.title,
contentType: `${app.name}:marketNews`,
requireValid: false,
parentPath: `${site_path}/${root}/${year}/${month}`,
data: item
})
addImageNode(newMarketNews);
// publish direct
if (newMarketNews) {
contentLib.publish(
{
keys: [newMarketNews._id],
sourceBranch: 'draft',
targetBranch: 'master'
}
)
}
return 1;
} catch (e) {
if (e.code === 'contentAlreadyExists') {
return 0;
} else {
log.error('Unexpected error: (%s) %s ', e.code, e.message);
return 0;
}
}
}
/*** HELPER FUNCTIONS ***/
function getFileName(url) {
return url.split('?')[0].split('/').pop();
}
/*** check if a folder exits ***/
function checkFolderExists(path) {
return contentLib.exists({key: path})
}
/***
* execute a function in admin context
* @param callback callback function to execute in the context
* @returns {*} whatever the function returns
*/
const runInContext = function (callback, repository) {
let result;
try {
result = contextLib.run({
principals: ["role:system.admin"],
user: {login: 'su'},
repository: repository,
}, callback);
} catch (e) {
log.info('Error: ' + e.message);
}
return result;
}