"use strict"; const sql = require('../services/sql'); const notes = require('../services/notes'); const axios = require('axios'); const log = require('../services/log'); const utils = require('../services/utils'); const unescape = require('unescape'); const attributes = require('../services/attributes'); const sync_mutex = require('../services/sync_mutex'); const config = require('../services/config'); const date_notes = require('../services/date_notes'); // "reddit" date note is subnote of date note which contains all reddit comments from that date const REDDIT_DATE_ATTRIBUTE = 'reddit_date_note'; async function createNote(parentNoteId, noteTitle, noteText) { return (await notes.createNewNote(parentNoteId, { note_title: noteTitle, note_text: noteText, target: 'into', is_protected: false })).noteId; } function redditId(kind, id) { return kind + "_" + id; } async function getDateNoteIdForReddit(dateTimeStr, rootNoteId) { const dateStr = dateTimeStr.substr(0, 10); let redditDateNoteId = await attributes.getNoteIdWithAttribute(REDDIT_DATE_ATTRIBUTE, dateStr); if (!redditDateNoteId) { const dateNoteId = await date_notes.getDateNoteId(dateTimeStr, rootNoteId); redditDateNoteId = await createNote(dateNoteId, "Reddit"); await attributes.createAttribute(redditDateNoteId, REDDIT_DATE_ATTRIBUTE, dateStr); } return redditDateNoteId; } async function importComments(rootNoteId, accountName, afterId = null) { let url = `https://www.reddit.com/user/${accountName}.json`; if (afterId) { url += "?after=" + afterId; } const response = await axios.get(url); const listing = response.data; if (listing.kind !== 'Listing') { log.info(`Reddit: Unknown object kind ${listing.kind}`); return; } const children = listing.data.children; let importedComments = 0; for (const child of children) { const comment = child.data; let commentNoteId = await attributes.getNoteIdWithAttribute('reddit_id', redditId(child.kind, comment.id)); if (commentNoteId) { continue; } const dateTimeStr = utils.dateStr(new Date(comment.created_utc * 1000)); const permaLink = 'https://reddit.com' + comment.permalink; const noteText = `
author: ${comment.author}, subreddit: ${comment.subreddit}, karma: ${comment.score}, created at ${dateTimeStr}
` + unescape(comment.body_html); let parentNoteId = await getDateNoteIdForReddit(dateTimeStr, rootNoteId); commentNoteId = await createNote(parentNoteId, comment.link_title, noteText); log.info("Reddit: Imported comment to note " + commentNoteId); importedComments++; await sql.doInTransaction(async () => { await attributes.createAttribute(commentNoteId, "reddit_kind", child.kind); await attributes.createAttribute(commentNoteId, "reddit_id", redditId(child.kind, comment.id)); await attributes.createAttribute(commentNoteId, "reddit_created_utc", comment.created_utc); }); } // if there have been no imported comments on this page, there shouldn't be any to import // on the next page since those are older if (listing.data.after && importedComments > 0) { importedComments += await importComments(rootNoteId, accountName, listing.data.after); } return importedComments; } let redditAccounts = []; async function runImport() { const rootNoteId = await date_notes.getRootNoteId(); // technically mutex shouldn't be necessary but we want to avoid doing potentially expensive import // concurrently with sync await sync_mutex.doExclusively(async () => { let importedComments = 0; for (const account of redditAccounts) { importedComments += await importComments(rootNoteId, account); } log.info(`Reddit: Imported ${importedComments} comments.`); }); } sql.dbReady.then(async () => { if (!config['Reddit'] || config['Reddit']['enabled'] !== true) { return; } const redditAccountsStr = config['Reddit']['accounts']; if (!redditAccountsStr) { log.info("Reddit: No reddit accounts defined in option 'reddit_accounts'"); } redditAccounts = redditAccountsStr.split(",").map(s => s.trim()); const pollingIntervalInSeconds = config['Reddit']['pollingIntervalInSeconds'] || (4 * 3600); setInterval(runImport, pollingIntervalInSeconds * 1000); setTimeout(runImport, 10000); // 10 seconds after startup - intentionally after initial sync });