utils.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107
  1. __filename__ = "utils.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.1.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import os
  9. import time
  10. import shutil
  11. import datetime
  12. import json
  13. from socket import error as SocketError
  14. import errno
  15. import urllib.request
  16. from pprint import pprint
  17. from calendar import monthrange
  18. from followingCalendar import addPersonToCalendar
  19. def getProtocolPrefixes() -> []:
  20. """Returns a list of valid prefixes
  21. """
  22. return ('https://', 'http://', 'dat://', 'i2p://', 'gnunet://',
  23. 'hyper://', 'gemini://', 'gopher://')
  24. def getLinkPrefixes() -> []:
  25. """Returns a list of valid web link prefixes
  26. """
  27. return ('https://', 'http://', 'dat://', 'i2p://', 'gnunet://',
  28. 'hyper://', 'gemini://', 'gopher://', 'briar:')
  29. def removeAvatarFromCache(baseDir: str, actorStr: str) -> None:
  30. """Removes any existing avatar entries from the cache
  31. This avoids duplicate entries with differing extensions
  32. """
  33. avatarFilenameExtensions = ('png', 'jpg', 'gif', 'webp')
  34. for extension in avatarFilenameExtensions:
  35. avatarFilename = \
  36. baseDir + '/cache/avatars/' + actorStr + '.' + extension
  37. if os.path.isfile(avatarFilename):
  38. os.remove(avatarFilename)
  39. def saveJson(jsonObject: {}, filename: str) -> bool:
  40. """Saves json to a file
  41. """
  42. tries = 0
  43. while tries < 5:
  44. try:
  45. with open(filename, 'w+') as fp:
  46. fp.write(json.dumps(jsonObject))
  47. return True
  48. except BaseException:
  49. print('WARN: saveJson ' + str(tries))
  50. time.sleep(1)
  51. tries += 1
  52. return False
  53. def loadJson(filename: str, delaySec=2) -> {}:
  54. """Makes a few attempts to load a json formatted file
  55. """
  56. jsonObject = None
  57. tries = 0
  58. while tries < 5:
  59. try:
  60. with open(filename, 'r') as fp:
  61. data = fp.read()
  62. jsonObject = json.loads(data)
  63. break
  64. except BaseException:
  65. print('WARN: loadJson exception')
  66. if delaySec > 0:
  67. time.sleep(delaySec)
  68. tries += 1
  69. return jsonObject
  70. def loadJsonOnionify(filename: str, domain: str, onionDomain: str,
  71. delaySec=2) -> {}:
  72. """Makes a few attempts to load a json formatted file
  73. This also converts the domain name to the onion domain
  74. """
  75. jsonObject = None
  76. tries = 0
  77. while tries < 5:
  78. try:
  79. with open(filename, 'r') as fp:
  80. data = fp.read()
  81. if data:
  82. data = data.replace(domain, onionDomain)
  83. data = data.replace('https:', 'http:')
  84. print('*****data: ' + data)
  85. jsonObject = json.loads(data)
  86. break
  87. except BaseException:
  88. print('WARN: loadJson exception')
  89. if delaySec > 0:
  90. time.sleep(delaySec)
  91. tries += 1
  92. return jsonObject
  93. def getStatusNumber() -> (str, str):
  94. """Returns the status number and published date
  95. """
  96. currTime = datetime.datetime.utcnow()
  97. daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days
  98. # status is the number of seconds since epoch
  99. statusNumber = \
  100. str(((daysSinceEpoch * 24 * 60 * 60) +
  101. (currTime.hour * 60 * 60) +
  102. (currTime.minute * 60) +
  103. currTime.second) * 1000 +
  104. int(currTime.microsecond / 1000))
  105. # See https://github.com/tootsuite/mastodon/blob/
  106. # 995f8b389a66ab76ec92d9a240de376f1fc13a38/lib/mastodon/snowflake.rb
  107. # use the leftover microseconds as the sequence number
  108. sequenceId = currTime.microsecond % 1000
  109. # shift by 16bits "sequence data"
  110. statusNumber = str((int(statusNumber) << 16) + sequenceId)
  111. published = currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  112. return statusNumber, published
  113. def evilIncarnate() -> []:
  114. return ('gab.com', 'gabfed.com', 'spinster.xyz',
  115. 'kiwifarms.cc', 'djitter.com')
  116. def isEvil(domain: str) -> bool:
  117. if not isinstance(domain, str):
  118. print('WARN: Malformed domain ' + str(domain))
  119. return True
  120. # https://www.youtube.com/watch?v=5qw1hcevmdU
  121. evilDomains = evilIncarnate()
  122. for concentratedEvil in evilDomains:
  123. if domain.endswith(concentratedEvil):
  124. return True
  125. return False
  126. def createPersonDir(nickname: str, domain: str, baseDir: str,
  127. dirname: str) -> str:
  128. """Create a directory for a person
  129. """
  130. handle = nickname + '@' + domain
  131. if not os.path.isdir(baseDir + '/accounts/' + handle):
  132. os.mkdir(baseDir + '/accounts/' + handle)
  133. boxDir = baseDir + '/accounts/' + handle + '/' + dirname
  134. if not os.path.isdir(boxDir):
  135. os.mkdir(boxDir)
  136. return boxDir
  137. def createOutboxDir(nickname: str, domain: str, baseDir: str) -> str:
  138. """Create an outbox for a person
  139. """
  140. return createPersonDir(nickname, domain, baseDir, 'outbox')
  141. def createInboxQueueDir(nickname: str, domain: str, baseDir: str) -> str:
  142. """Create an inbox queue and returns the feed filename and directory
  143. """
  144. return createPersonDir(nickname, domain, baseDir, 'queue')
  145. def domainPermitted(domain: str, federationList: []):
  146. if len(federationList) == 0:
  147. return True
  148. if ':' in domain:
  149. domain = domain.split(':')[0]
  150. if domain in federationList:
  151. return True
  152. return False
  153. def urlPermitted(url: str, federationList: [], capability: str):
  154. if isEvil(url):
  155. return False
  156. if not federationList:
  157. return True
  158. for domain in federationList:
  159. if domain in url:
  160. return True
  161. return False
  162. def getDisplayName(baseDir: str, actor: str, personCache: {}) -> str:
  163. """Returns the display name for the given actor
  164. """
  165. if '/statuses/' in actor:
  166. actor = actor.split('/statuses/')[0]
  167. if not personCache.get(actor):
  168. return None
  169. if personCache[actor].get('actor'):
  170. if personCache[actor]['actor'].get('name'):
  171. return personCache[actor]['actor']['name']
  172. else:
  173. # Try to obtain from the cached actors
  174. cachedActorFilename = \
  175. baseDir + '/cache/actors/' + (actor.replace('/', '#')) + '.json'
  176. if os.path.isfile(cachedActorFilename):
  177. actorJson = loadJson(cachedActorFilename, 1)
  178. if actorJson:
  179. if actorJson.get('name'):
  180. return(actorJson['name'])
  181. return None
  182. def getNicknameFromActor(actor: str) -> str:
  183. """Returns the nickname from an actor url
  184. """
  185. if '/users/' not in actor:
  186. if '/profile/' in actor:
  187. nickStr = actor.split('/profile/')[1].replace('@', '')
  188. if '/' not in nickStr:
  189. return nickStr
  190. else:
  191. return nickStr.split('/')[0]
  192. if '/channel/' in actor:
  193. nickStr = actor.split('/channel/')[1].replace('@', '')
  194. if '/' not in nickStr:
  195. return nickStr
  196. else:
  197. return nickStr.split('/')[0]
  198. # https://domain/@nick
  199. if '/@' in actor:
  200. nickStr = actor.split('/@')[1]
  201. if '/' in nickStr:
  202. nickStr = nickStr.split('/')[0]
  203. return nickStr
  204. return None
  205. nickStr = actor.split('/users/')[1].replace('@', '')
  206. if '/' not in nickStr:
  207. return nickStr
  208. else:
  209. return nickStr.split('/')[0]
  210. def getDomainFromActor(actor: str) -> (str, int):
  211. """Returns the domain name from an actor url
  212. """
  213. port = None
  214. prefixes = getProtocolPrefixes()
  215. if '/profile/' in actor:
  216. domain = actor.split('/profile/')[0]
  217. for prefix in prefixes:
  218. domain = domain.replace(prefix, '')
  219. else:
  220. if '/channel/' in actor:
  221. domain = actor.split('/channel/')[0]
  222. for prefix in prefixes:
  223. domain = domain.replace(prefix, '')
  224. else:
  225. if '/users/' not in actor:
  226. domain = actor
  227. for prefix in prefixes:
  228. domain = domain.replace(prefix, '')
  229. if '/' in actor:
  230. domain = domain.split('/')[0]
  231. else:
  232. domain = actor.split('/users/')[0]
  233. for prefix in prefixes:
  234. domain = domain.replace(prefix, '')
  235. if ':' in domain:
  236. portStr = domain.split(':')[1]
  237. if not portStr.isdigit():
  238. return None, None
  239. port = int(portStr)
  240. domain = domain.split(':')[0]
  241. return domain, port
  242. def followPerson(baseDir: str, nickname: str, domain: str,
  243. followNickname: str, followDomain: str,
  244. federationList: [], debug: bool,
  245. followFile='following.txt') -> bool:
  246. """Adds a person to the follow list
  247. """
  248. if not domainPermitted(followDomain.lower().replace('\n', ''),
  249. federationList):
  250. if debug:
  251. print('DEBUG: follow of domain ' +
  252. followDomain + ' not permitted')
  253. return False
  254. if debug:
  255. print('DEBUG: follow of domain ' + followDomain)
  256. if ':' in domain:
  257. handle = nickname + '@' + domain.split(':')[0].lower()
  258. else:
  259. handle = nickname + '@' + domain.lower()
  260. if not os.path.isdir(baseDir + '/accounts/' + handle):
  261. print('WARN: account for ' + handle + ' does not exist')
  262. return False
  263. if ':' in followDomain:
  264. handleToFollow = followNickname + '@' + followDomain.split(':')[0]
  265. else:
  266. handleToFollow = followNickname + '@' + followDomain
  267. # was this person previously unfollowed?
  268. unfollowedFilename = baseDir + '/accounts/' + handle + '/unfollowed.txt'
  269. if os.path.isfile(unfollowedFilename):
  270. if handleToFollow in open(unfollowedFilename).read():
  271. # remove them from the unfollowed file
  272. newLines = ''
  273. with open(unfollowedFilename, "r") as f:
  274. lines = f.readlines()
  275. for line in lines:
  276. if handleToFollow not in line:
  277. newLines += line
  278. with open(unfollowedFilename, "w") as f:
  279. f.write(newLines)
  280. if not os.path.isdir(baseDir + '/accounts'):
  281. os.mkdir(baseDir + '/accounts')
  282. handleToFollow = followNickname + '@' + followDomain
  283. filename = baseDir + '/accounts/' + handle + '/' + followFile
  284. if os.path.isfile(filename):
  285. if handleToFollow in open(filename).read():
  286. if debug:
  287. print('DEBUG: follow already exists')
  288. return True
  289. # prepend to follow file
  290. try:
  291. with open(filename, 'r+') as followFile:
  292. content = followFile.read()
  293. followFile.seek(0, 0)
  294. followFile.write(handleToFollow + '\n' + content)
  295. if debug:
  296. print('DEBUG: follow added')
  297. return True
  298. except Exception as e:
  299. print('WARN: Failed to write entry to follow file ' +
  300. filename + ' ' + str(e))
  301. if followFile == 'following.txt':
  302. # if following a person add them to the list of
  303. # calendar follows
  304. addPersonToCalendar(baseDir, nickname, domain,
  305. followNickname, followDomain)
  306. if debug:
  307. print('DEBUG: creating new following file to follow ' + handleToFollow)
  308. with open(filename, "w") as followfile:
  309. followfile.write(handleToFollow + '\n')
  310. return True
  311. def locatePost(baseDir: str, nickname: str, domain: str,
  312. postUrl: str, replies=False) -> str:
  313. """Returns the filename for the given status post url
  314. """
  315. if not replies:
  316. extension = 'json'
  317. else:
  318. extension = 'replies'
  319. # if this post in the shared inbox?
  320. postUrl = postUrl.replace('/', '#').replace('/activity', '').strip()
  321. # add the extension
  322. postUrl = postUrl + '.' + extension
  323. # search boxes
  324. boxes = ('inbox', 'outbox', 'tlblogs')
  325. accountDir = baseDir + '/accounts/' + nickname + '@' + domain + '/'
  326. for boxName in boxes:
  327. postFilename = accountDir + boxName + '/' + postUrl
  328. if os.path.isfile(postFilename):
  329. return postFilename
  330. # is it in the announce cache?
  331. postFilename = baseDir + '/cache/announce/' + nickname + '/' + postUrl
  332. if os.path.isfile(postFilename):
  333. return postFilename
  334. print('WARN: unable to locate ' + nickname + ' ' + postUrl)
  335. return None
  336. def removeAttachment(baseDir: str, httpPrefix: str, domain: str, postJson: {}):
  337. if not postJson.get('attachment'):
  338. return
  339. if not postJson['attachment'][0].get('url'):
  340. return
  341. # if port:
  342. # if port != 80 and port != 443:
  343. # if ':' not in domain:
  344. # domain = domain + ':' + str(port)
  345. attachmentUrl = postJson['attachment'][0]['url']
  346. if not attachmentUrl:
  347. return
  348. mediaFilename = baseDir + '/' + \
  349. attachmentUrl.replace(httpPrefix + '://' + domain + '/', '')
  350. if os.path.isfile(mediaFilename):
  351. os.remove(mediaFilename)
  352. etagFilename = mediaFilename + '.etag'
  353. if os.path.isfile(etagFilename):
  354. os.remove(etagFilename)
  355. postJson['attachment'] = []
  356. def removeModerationPostFromIndex(baseDir: str, postUrl: str,
  357. debug: bool) -> None:
  358. """Removes a url from the moderation index
  359. """
  360. moderationIndexFile = baseDir + '/accounts/moderation.txt'
  361. if not os.path.isfile(moderationIndexFile):
  362. return
  363. postId = postUrl.replace('/activity', '')
  364. if postId in open(moderationIndexFile).read():
  365. with open(moderationIndexFile, "r") as f:
  366. lines = f.readlines()
  367. with open(moderationIndexFile, "w+") as f:
  368. for line in lines:
  369. if line.strip("\n").strip("\r") != postId:
  370. f.write(line)
  371. else:
  372. if debug:
  373. print('DEBUG: removed ' + postId +
  374. ' from moderation index')
  375. def isReplyToBlogPost(baseDir: str, nickname: str, domain: str,
  376. postJsonObject: str):
  377. """Is the given post a reply to a blog post?
  378. """
  379. if not postJsonObject.get('object'):
  380. return False
  381. if not isinstance(postJsonObject['object'], dict):
  382. return False
  383. if not postJsonObject['object'].get('inReplyTo'):
  384. return False
  385. blogsIndexFilename = baseDir + '/accounts/' + \
  386. nickname + '@' + domain + '/tlblogs.index'
  387. if not os.path.isfile(blogsIndexFilename):
  388. return False
  389. postId = postJsonObject['object']['inReplyTo'].replace('/activity', '')
  390. postId = postId.replace('/', '#')
  391. if postId in open(blogsIndexFilename).read():
  392. return True
  393. return False
  394. def deletePost(baseDir: str, httpPrefix: str,
  395. nickname: str, domain: str, postFilename: str,
  396. debug: bool, recentPostsCache: {}) -> None:
  397. """Recursively deletes a post and its replies and attachments
  398. """
  399. postJsonObject = loadJson(postFilename, 1)
  400. if postJsonObject:
  401. # don't allow deletion of bookmarked posts
  402. bookmarksIndexFilename = \
  403. baseDir + '/accounts/' + nickname + '@' + domain + \
  404. '/bookmarks.index'
  405. if os.path.isfile(bookmarksIndexFilename):
  406. bookmarkIndex = postFilename.split('/')[-1] + '\n'
  407. if bookmarkIndex in open(bookmarksIndexFilename).read():
  408. return
  409. # don't remove replies to blog posts
  410. if isReplyToBlogPost(baseDir, nickname, domain,
  411. postJsonObject):
  412. return
  413. # remove from recent posts cache in memory
  414. if recentPostsCache:
  415. postId = \
  416. postJsonObject['id'].replace('/activity', '').replace('/', '#')
  417. if recentPostsCache.get('index'):
  418. if postId in recentPostsCache['index']:
  419. recentPostsCache['index'].remove(postId)
  420. if recentPostsCache['json'].get(postId):
  421. del recentPostsCache['json'][postId]
  422. # remove any attachment
  423. removeAttachment(baseDir, httpPrefix, domain, postJsonObject)
  424. # remove any mute file
  425. muteFilename = postFilename + '.muted'
  426. if os.path.isfile(muteFilename):
  427. os.remove(muteFilename)
  428. # remove cached html version of the post
  429. cachedPostFilename = \
  430. getCachedPostFilename(baseDir, nickname, domain, postJsonObject)
  431. if cachedPostFilename:
  432. if os.path.isfile(cachedPostFilename):
  433. os.remove(cachedPostFilename)
  434. # removePostFromCache(postJsonObject,recentPostsCache)
  435. hasObject = False
  436. if postJsonObject.get('object'):
  437. hasObject = True
  438. # remove from moderation index file
  439. if hasObject:
  440. if isinstance(postJsonObject['object'], dict):
  441. if postJsonObject['object'].get('moderationStatus'):
  442. if postJsonObject.get('id'):
  443. postId = postJsonObject['id'].replace('/activity', '')
  444. removeModerationPostFromIndex(baseDir, postId, debug)
  445. # remove any hashtags index entries
  446. removeHashtagIndex = False
  447. if hasObject:
  448. if hasObject and isinstance(postJsonObject['object'], dict):
  449. if postJsonObject['object'].get('content'):
  450. if '#' in postJsonObject['object']['content']:
  451. removeHashtagIndex = True
  452. if removeHashtagIndex:
  453. if postJsonObject['object'].get('id') and \
  454. postJsonObject['object'].get('tag'):
  455. # get the id of the post
  456. postId = \
  457. postJsonObject['object']['id'].replace('/activity', '')
  458. for tag in postJsonObject['object']['tag']:
  459. if tag['type'] != 'Hashtag':
  460. continue
  461. if not tag.get('name'):
  462. continue
  463. # find the index file for this tag
  464. tagIndexFilename = \
  465. baseDir + '/tags/' + tag['name'][1:] + '.txt'
  466. if not os.path.isfile(tagIndexFilename):
  467. continue
  468. # remove postId from the tag index file
  469. lines = None
  470. with open(tagIndexFilename, "r") as f:
  471. lines = f.readlines()
  472. if lines:
  473. newlines = ''
  474. for fileLine in lines:
  475. if postId in fileLine:
  476. continue
  477. newlines += fileLine
  478. if not newlines.strip():
  479. # if there are no lines then remove the
  480. # hashtag file
  481. os.remove(tagIndexFilename)
  482. else:
  483. with open(tagIndexFilename, "w+") as f:
  484. f.write(newlines)
  485. # remove any replies
  486. repliesFilename = postFilename.replace('.json', '.replies')
  487. if os.path.isfile(repliesFilename):
  488. if debug:
  489. print('DEBUG: removing replies to ' + postFilename)
  490. with open(repliesFilename, 'r') as f:
  491. for replyId in f:
  492. replyFile = locatePost(baseDir, nickname, domain, replyId)
  493. if replyFile:
  494. if os.path.isfile(replyFile):
  495. deletePost(baseDir, httpPrefix,
  496. nickname, domain, replyFile, debug,
  497. recentPostsCache)
  498. # remove the replies file
  499. os.remove(repliesFilename)
  500. # finally, remove the post itself
  501. os.remove(postFilename)
  502. def validNickname(domain: str, nickname: str) -> bool:
  503. forbiddenChars = ('.', ' ', '/', '?', ':', ';', '@')
  504. for c in forbiddenChars:
  505. if c in nickname:
  506. return False
  507. if nickname == domain:
  508. return False
  509. reservedNames = ('inbox', 'dm', 'outbox', 'following',
  510. 'public', 'followers', 'profile',
  511. 'channel', 'capabilities', 'calendar',
  512. 'tlreplies', 'tlmedia', 'tlblogs',
  513. 'moderation', 'activity', 'undo',
  514. 'reply', 'replies', 'question', 'like',
  515. 'likes', 'users', 'statuses',
  516. 'updates', 'repeat', 'announce',
  517. 'shares', 'fonts', 'icons')
  518. if nickname in reservedNames:
  519. return False
  520. return True
  521. def noOfAccounts(baseDir: str) -> bool:
  522. """Returns the number of accounts on the system
  523. """
  524. accountCtr = 0
  525. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  526. for account in dirs:
  527. if '@' in account:
  528. if not account.startswith('inbox@'):
  529. accountCtr += 1
  530. return accountCtr
  531. def noOfActiveAccountsMonthly(baseDir: str, months: int) -> bool:
  532. """Returns the number of accounts on the system this month
  533. """
  534. accountCtr = 0
  535. currTime = int(time.time())
  536. monthSeconds = int(60*60*24*30*months)
  537. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  538. for account in dirs:
  539. if '@' in account:
  540. if not account.startswith('inbox@'):
  541. lastUsedFilename = \
  542. baseDir + '/accounts/' + account + '/.lastUsed'
  543. if os.path.isfile(lastUsedFilename):
  544. with open(lastUsedFilename, 'r') as lastUsedFile:
  545. lastUsed = lastUsedFile.read()
  546. if lastUsed.isdigit():
  547. timeDiff = (currTime - int(lastUsed))
  548. if timeDiff < monthSeconds:
  549. accountCtr += 1
  550. return accountCtr
  551. def isPublicPostFromUrl(baseDir: str, nickname: str, domain: str,
  552. postUrl: str) -> bool:
  553. """Returns whether the given url is a public post
  554. """
  555. postFilename = locatePost(baseDir, nickname, domain, postUrl)
  556. if not postFilename:
  557. return False
  558. postJsonObject = loadJson(postFilename, 1)
  559. if not postJsonObject:
  560. return False
  561. return isPublicPost(postJsonObject)
  562. def isPublicPost(postJsonObject: {}) -> bool:
  563. """Returns true if the given post is public
  564. """
  565. if not postJsonObject.get('type'):
  566. return False
  567. if postJsonObject['type'] != 'Create':
  568. return False
  569. if not postJsonObject.get('object'):
  570. return False
  571. if not isinstance(postJsonObject['object'], dict):
  572. return False
  573. if not postJsonObject['object'].get('to'):
  574. return False
  575. for recipient in postJsonObject['object']['to']:
  576. if recipient.endswith('#Public'):
  577. return True
  578. return False
  579. def copytree(src: str, dst: str, symlinks=False, ignore=None):
  580. """Copy a directory
  581. """
  582. for item in os.listdir(src):
  583. s = os.path.join(src, item)
  584. d = os.path.join(dst, item)
  585. if os.path.isdir(s):
  586. shutil.copytree(s, d, symlinks, ignore)
  587. else:
  588. shutil.copy2(s, d)
  589. def getCachedPostDirectory(baseDir: str, nickname: str, domain: str) -> str:
  590. """Returns the directory where the html post cache exists
  591. """
  592. htmlPostCacheDir = baseDir + '/accounts/' + \
  593. nickname + '@' + domain + '/postcache'
  594. return htmlPostCacheDir
  595. def getCachedPostFilename(baseDir: str, nickname: str, domain: str,
  596. postJsonObject: {}) -> str:
  597. """Returns the html cache filename for the given post
  598. """
  599. cachedPostDir = getCachedPostDirectory(baseDir, nickname, domain)
  600. if not os.path.isdir(cachedPostDir):
  601. # print('ERROR: invalid html cache directory '+cachedPostDir)
  602. return None
  603. if '@' not in cachedPostDir:
  604. # print('ERROR: invalid html cache directory '+cachedPostDir)
  605. return None
  606. cachedPostFilename = \
  607. cachedPostDir + \
  608. '/' + postJsonObject['id'].replace('/activity', '').replace('/', '#')
  609. cachedPostFilename = cachedPostFilename + '.html'
  610. return cachedPostFilename
  611. def removePostFromCache(postJsonObject: {}, recentPostsCache: {}):
  612. """ if the post exists in the recent posts cache then remove it
  613. """
  614. if not postJsonObject.get('id'):
  615. return
  616. if not recentPostsCache.get('index'):
  617. return
  618. postId = postJsonObject['id']
  619. if '#' in postId:
  620. postId = postId.split('#', 1)[0]
  621. postId = postId.replace('/activity', '').replace('/', '#')
  622. if postId not in recentPostsCache['index']:
  623. return
  624. if recentPostsCache['json'].get(postId):
  625. del recentPostsCache['json'][postId]
  626. if recentPostsCache['html'].get(postId):
  627. del recentPostsCache['html'][postId]
  628. recentPostsCache['index'].remove(postId)
  629. def updateRecentPostsCache(recentPostsCache: {}, maxRecentPosts: int,
  630. postJsonObject: {}, htmlStr: str) -> None:
  631. """Store recent posts in memory so that they can be quickly recalled
  632. """
  633. if not postJsonObject.get('id'):
  634. return
  635. postId = postJsonObject['id']
  636. if '#' in postId:
  637. postId = postId.split('#', 1)[0]
  638. postId = postId.replace('/activity', '').replace('/', '#')
  639. if recentPostsCache.get('index'):
  640. if postId in recentPostsCache['index']:
  641. return
  642. recentPostsCache['index'].append(postId)
  643. postJsonObject['muted'] = False
  644. recentPostsCache['json'][postId] = json.dumps(postJsonObject)
  645. recentPostsCache['html'][postId] = htmlStr
  646. while len(recentPostsCache['html'].items()) > maxRecentPosts:
  647. recentPostsCache['index'].pop(0)
  648. del recentPostsCache['json'][postId]
  649. del recentPostsCache['html'][postId]
  650. else:
  651. recentPostsCache['index'] = [postId]
  652. recentPostsCache['json'] = {}
  653. recentPostsCache['html'] = {}
  654. recentPostsCache['json'][postId] = json.dumps(postJsonObject)
  655. recentPostsCache['html'][postId] = htmlStr
  656. def fileLastModified(filename: str) -> str:
  657. """Returns the date when a file was last modified
  658. """
  659. t = os.path.getmtime(filename)
  660. modifiedTime = datetime.datetime.fromtimestamp(t)
  661. return modifiedTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  662. def daysInMonth(year: int, monthNumber: int) -> int:
  663. """Returns the number of days in the month
  664. """
  665. if monthNumber < 1 or monthNumber > 12:
  666. return None
  667. daysRange = monthrange(year, monthNumber)
  668. return daysRange[1]
  669. def mergeDicts(dict1: {}, dict2: {}) -> {}:
  670. """Merges two dictionaries
  671. """
  672. res = {**dict1, **dict2}
  673. return res
  674. def isBlogPost(postJsonObject: {}) -> bool:
  675. """Is the given post a blog post?
  676. """
  677. if postJsonObject['type'] != 'Create':
  678. return False
  679. if not postJsonObject.get('object'):
  680. return False
  681. if not isinstance(postJsonObject['object'], dict):
  682. return False
  683. if not postJsonObject['object'].get('type'):
  684. return False
  685. if not postJsonObject['object'].get('content'):
  686. return False
  687. if postJsonObject['object']['type'] != 'Article':
  688. return False
  689. return True
  690. def searchBoxPosts(baseDir: str, nickname: str, domain: str,
  691. searchStr: str, maxResults: int,
  692. boxName='outbox') -> []:
  693. """Search your posts and return a list of the filenames
  694. containing matching strings
  695. """
  696. path = baseDir + '/accounts/' + nickname + '@' + domain + '/' + boxName
  697. if not os.path.isdir(path):
  698. return []
  699. searchStr = searchStr.lower().strip()
  700. if '+' in searchStr:
  701. searchWords = searchStr.split('+')
  702. for index in range(len(searchWords)):
  703. searchWords[index] = searchWords[index].strip()
  704. print('SEARCH: ' + str(searchWords))
  705. else:
  706. searchWords = [searchStr]
  707. res = []
  708. for root, dirs, fnames in os.walk(path):
  709. for fname in fnames:
  710. filePath = os.path.join(root, fname)
  711. with open(filePath, 'r') as postFile:
  712. data = postFile.read().lower()
  713. notFound = False
  714. for keyword in searchWords:
  715. if keyword not in data:
  716. notFound = True
  717. break
  718. if notFound:
  719. continue
  720. res.append(filePath)
  721. if len(res) >= maxResults:
  722. return res
  723. return res
  724. def getFileCaseInsensitive(path: str) -> str:
  725. """Returns a case specific filename given a case insensitive version of it
  726. """
  727. directory, filename = os.path.split(path)
  728. directory, filename = (directory or '.'), filename.lower()
  729. for f in os.listdir(directory):
  730. newpath = os.path.join(directory, f)
  731. if os.path.isfile(newpath) and f.lower() == filename:
  732. return newpath
  733. return path
  734. def undoLikesCollectionEntry(recentPostsCache: {},
  735. baseDir: str, postFilename: str, objectUrl: str,
  736. actor: str, domain: str, debug: bool) -> None:
  737. """Undoes a like for a particular actor
  738. """
  739. postJsonObject = loadJson(postFilename)
  740. if postJsonObject:
  741. # remove any cached version of this post so that the
  742. # like icon is changed
  743. nickname = getNicknameFromActor(actor)
  744. cachedPostFilename = getCachedPostFilename(baseDir, nickname,
  745. domain, postJsonObject)
  746. if cachedPostFilename:
  747. if os.path.isfile(cachedPostFilename):
  748. os.remove(cachedPostFilename)
  749. removePostFromCache(postJsonObject, recentPostsCache)
  750. if not postJsonObject.get('type'):
  751. return
  752. if postJsonObject['type'] != 'Create':
  753. return
  754. if not postJsonObject.get('object'):
  755. if debug:
  756. pprint(postJsonObject)
  757. print('DEBUG: post '+objectUrl+' has no object')
  758. return
  759. if not isinstance(postJsonObject['object'], dict):
  760. return
  761. if not postJsonObject['object'].get('likes'):
  762. return
  763. if not isinstance(postJsonObject['object']['likes'], dict):
  764. return
  765. if not postJsonObject['object']['likes'].get('items'):
  766. return
  767. totalItems = 0
  768. if postJsonObject['object']['likes'].get('totalItems'):
  769. totalItems = postJsonObject['object']['likes']['totalItems']
  770. itemFound = False
  771. for likeItem in postJsonObject['object']['likes']['items']:
  772. if likeItem.get('actor'):
  773. if likeItem['actor'] == actor:
  774. if debug:
  775. print('DEBUG: like was removed for ' + actor)
  776. postJsonObject['object']['likes']['items'].remove(likeItem)
  777. itemFound = True
  778. break
  779. if itemFound:
  780. if totalItems == 1:
  781. if debug:
  782. print('DEBUG: likes was removed from post')
  783. del postJsonObject['object']['likes']
  784. else:
  785. itlen = len(postJsonObject['object']['likes']['items'])
  786. postJsonObject['object']['likes']['totalItems'] = itlen
  787. saveJson(postJsonObject, postFilename)
  788. def updateLikesCollection(recentPostsCache: {},
  789. baseDir: str, postFilename: str,
  790. objectUrl: str,
  791. actor: str, domain: str, debug: bool) -> None:
  792. """Updates the likes collection within a post
  793. """
  794. postJsonObject = loadJson(postFilename)
  795. if not postJsonObject:
  796. return
  797. # remove any cached version of this post so that the
  798. # like icon is changed
  799. nickname = getNicknameFromActor(actor)
  800. cachedPostFilename = getCachedPostFilename(baseDir, nickname,
  801. domain, postJsonObject)
  802. if cachedPostFilename:
  803. if os.path.isfile(cachedPostFilename):
  804. os.remove(cachedPostFilename)
  805. removePostFromCache(postJsonObject, recentPostsCache)
  806. if not postJsonObject.get('object'):
  807. if debug:
  808. pprint(postJsonObject)
  809. print('DEBUG: post ' + objectUrl + ' has no object')
  810. return
  811. if not isinstance(postJsonObject['object'], dict):
  812. return
  813. if not objectUrl.endswith('/likes'):
  814. objectUrl = objectUrl + '/likes'
  815. if not postJsonObject['object'].get('likes'):
  816. if debug:
  817. print('DEBUG: Adding initial like to ' + objectUrl)
  818. likesJson = {
  819. "@context": "https://www.w3.org/ns/activitystreams",
  820. 'id': objectUrl,
  821. 'type': 'Collection',
  822. "totalItems": 1,
  823. 'items': [{
  824. 'type': 'Like',
  825. 'actor': actor
  826. }]
  827. }
  828. postJsonObject['object']['likes'] = likesJson
  829. else:
  830. if not postJsonObject['object']['likes'].get('items'):
  831. postJsonObject['object']['likes']['items'] = []
  832. for likeItem in postJsonObject['object']['likes']['items']:
  833. if likeItem.get('actor'):
  834. if likeItem['actor'] == actor:
  835. return
  836. newLike = {
  837. 'type': 'Like',
  838. 'actor': actor
  839. }
  840. postJsonObject['object']['likes']['items'].append(newLike)
  841. itlen = len(postJsonObject['object']['likes']['items'])
  842. postJsonObject['object']['likes']['totalItems'] = itlen
  843. if debug:
  844. print('DEBUG: saving post with likes added')
  845. pprint(postJsonObject)
  846. saveJson(postJsonObject, postFilename)
  847. def undoAnnounceCollectionEntry(recentPostsCache: {},
  848. baseDir: str, postFilename: str,
  849. actor: str, domain: str, debug: bool) -> None:
  850. """Undoes an announce for a particular actor by removing it from
  851. the "shares" collection within a post. Note that the "shares"
  852. collection has no relation to shared items in shares.py. It's
  853. shares of posts, not shares of physical objects.
  854. """
  855. postJsonObject = loadJson(postFilename)
  856. if postJsonObject:
  857. # remove any cached version of this announce so that the announce
  858. # icon is changed
  859. nickname = getNicknameFromActor(actor)
  860. cachedPostFilename = getCachedPostFilename(baseDir, nickname, domain,
  861. postJsonObject)
  862. if cachedPostFilename:
  863. if os.path.isfile(cachedPostFilename):
  864. os.remove(cachedPostFilename)
  865. removePostFromCache(postJsonObject, recentPostsCache)
  866. if not postJsonObject.get('type'):
  867. return
  868. if postJsonObject['type'] != 'Create':
  869. return
  870. if not postJsonObject.get('object'):
  871. if debug:
  872. pprint(postJsonObject)
  873. print('DEBUG: post has no object')
  874. return
  875. if not isinstance(postJsonObject['object'], dict):
  876. return
  877. if not postJsonObject['object'].get('shares'):
  878. return
  879. if not postJsonObject['object']['shares'].get('items'):
  880. return
  881. totalItems = 0
  882. if postJsonObject['object']['shares'].get('totalItems'):
  883. totalItems = postJsonObject['object']['shares']['totalItems']
  884. itemFound = False
  885. for announceItem in postJsonObject['object']['shares']['items']:
  886. if announceItem.get('actor'):
  887. if announceItem['actor'] == actor:
  888. if debug:
  889. print('DEBUG: Announce was removed for ' + actor)
  890. anIt = announceItem
  891. postJsonObject['object']['shares']['items'].remove(anIt)
  892. itemFound = True
  893. break
  894. if itemFound:
  895. if totalItems == 1:
  896. if debug:
  897. print('DEBUG: shares (announcements) ' +
  898. 'was removed from post')
  899. del postJsonObject['object']['shares']
  900. else:
  901. itlen = len(postJsonObject['object']['shares']['items'])
  902. postJsonObject['object']['shares']['totalItems'] = itlen
  903. saveJson(postJsonObject, postFilename)
  904. def updateAnnounceCollection(recentPostsCache: {},
  905. baseDir: str, postFilename: str,
  906. actor: str, domain: str, debug: bool) -> None:
  907. """Updates the announcements collection within a post
  908. Confusingly this is known as "shares", but isn't the
  909. same as shared items within shares.py
  910. It's shares of posts, not shares of physical objects.
  911. """
  912. postJsonObject = loadJson(postFilename)
  913. if postJsonObject:
  914. # remove any cached version of this announce so that the announce
  915. # icon is changed
  916. nickname = getNicknameFromActor(actor)
  917. cachedPostFilename = getCachedPostFilename(baseDir, nickname, domain,
  918. postJsonObject)
  919. if cachedPostFilename:
  920. if os.path.isfile(cachedPostFilename):
  921. os.remove(cachedPostFilename)
  922. removePostFromCache(postJsonObject, recentPostsCache)
  923. if not postJsonObject.get('object'):
  924. if debug:
  925. pprint(postJsonObject)
  926. print('DEBUG: post ' + postFilename + ' has no object')
  927. return
  928. if not isinstance(postJsonObject['object'], dict):
  929. return
  930. postUrl = postJsonObject['id'].replace('/activity', '') + '/shares'
  931. if not postJsonObject['object'].get('shares'):
  932. if debug:
  933. print('DEBUG: Adding initial shares (announcements) to ' +
  934. postUrl)
  935. announcementsJson = {
  936. "@context": "https://www.w3.org/ns/activitystreams",
  937. 'id': postUrl,
  938. 'type': 'Collection',
  939. "totalItems": 1,
  940. 'items': [{
  941. 'type': 'Announce',
  942. 'actor': actor
  943. }]
  944. }
  945. postJsonObject['object']['shares'] = announcementsJson
  946. else:
  947. if postJsonObject['object']['shares'].get('items'):
  948. sharesItems = postJsonObject['object']['shares']['items']
  949. for announceItem in sharesItems:
  950. if announceItem.get('actor'):
  951. if announceItem['actor'] == actor:
  952. return
  953. newAnnounce = {
  954. 'type': 'Announce',
  955. 'actor': actor
  956. }
  957. postJsonObject['object']['shares']['items'].append(newAnnounce)
  958. itlen = len(postJsonObject['object']['shares']['items'])
  959. postJsonObject['object']['shares']['totalItems'] = itlen
  960. else:
  961. if debug:
  962. print('DEBUG: shares (announcements) section of post ' +
  963. 'has no items list')
  964. if debug:
  965. print('DEBUG: saving post with shares (announcements) added')
  966. pprint(postJsonObject)
  967. saveJson(postJsonObject, postFilename)
  968. def siteIsActive(url: str) -> bool:
  969. """Returns true if the current url is resolvable.
  970. This can be used to check that an instance is online before
  971. trying to send posts to it.
  972. """
  973. if not url.startswith('http'):
  974. return False
  975. try:
  976. req = urllib.request.Request(url)
  977. urllib.request.urlopen(req, timeout=10) # nosec
  978. return True
  979. except SocketError as e:
  980. if e.errno == errno.ECONNRESET:
  981. print('WARN: connection was reset during siteIsActive')
  982. return False