posts.py 107 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import requests
  9. import json
  10. import html
  11. import datetime
  12. import os
  13. import shutil
  14. import threading
  15. import sys
  16. import trace
  17. import time
  18. from time import gmtime, strftime
  19. from collections import OrderedDict
  20. from threads import threadWithTrace
  21. from cache import storePersonInCache
  22. from cache import getPersonFromCache
  23. from cache import expirePersonCache
  24. from pprint import pprint
  25. from random import randint
  26. from session import createSession
  27. from session import getJson
  28. from session import postJsonString
  29. from session import postImage
  30. from webfinger import webfingerHandle
  31. from httpsig import createSignedHeader
  32. from utils import removePostFromCache
  33. from utils import getCachedPostFilename
  34. from utils import getStatusNumber
  35. from utils import createPersonDir
  36. from utils import urlPermitted
  37. from utils import getNicknameFromActor
  38. from utils import getDomainFromActor
  39. from utils import deletePost
  40. from utils import validNickname
  41. from utils import locatePost
  42. from utils import loadJson
  43. from utils import saveJson
  44. from capabilities import getOcapFilename
  45. from capabilities import capabilitiesUpdate
  46. from media import attachMedia
  47. from content import addHtmlTags
  48. from content import replaceEmojiFromTags
  49. from auth import createBasicAuthHeader
  50. from config import getConfigParam
  51. from blocking import isBlocked
  52. try:
  53. from BeautifulSoup import BeautifulSoup
  54. except ImportError:
  55. from bs4 import BeautifulSoup
  56. def isModerator(baseDir: str,nickname: str) -> bool:
  57. """Returns true if the given nickname is a moderator
  58. """
  59. moderatorsFile=baseDir+'/accounts/moderators.txt'
  60. if not os.path.isfile(moderatorsFile):
  61. if getConfigParam(baseDir,'admin')==nickname:
  62. return True
  63. return False
  64. with open(moderatorsFile, "r") as f:
  65. lines = f.readlines()
  66. if len(lines)==0:
  67. if getConfigParam(baseDir,'admin')==nickname:
  68. return True
  69. for moderator in lines:
  70. moderator=moderator.strip('\n')
  71. if moderator==nickname:
  72. return True
  73. return False
  74. def noOfFollowersOnDomain(baseDir: str,handle: str, \
  75. domain: str, followFile='followers.txt') -> int:
  76. """Returns the number of followers of the given handle from the given domain
  77. """
  78. filename=baseDir+'/accounts/'+handle+'/'+followFile
  79. if not os.path.isfile(filename):
  80. return 0
  81. ctr=0
  82. with open(filename, "r") as followersFilename:
  83. for followerHandle in followersFilename:
  84. if '@' in followerHandle:
  85. followerDomain= \
  86. followerHandle.split('@')[1].replace('\n','')
  87. if domain==followerDomain:
  88. ctr+=1
  89. return ctr
  90. def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
  91. debug=False):
  92. """Returns the public or private key of a person
  93. """
  94. handle=nickname+'@'+domain
  95. keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
  96. if not os.path.isfile(keyFilename):
  97. if debug:
  98. print('DEBUG: private key file not found: '+keyFilename)
  99. return ''
  100. keyPem=''
  101. with open(keyFilename, "r") as pemFile:
  102. keyPem=pemFile.read()
  103. if len(keyPem)<20:
  104. if debug:
  105. print('DEBUG: private key was too short: '+keyPem)
  106. return ''
  107. return keyPem
  108. def cleanHtml(rawHtml: str) -> str:
  109. text = BeautifulSoup(rawHtml, 'html.parser').get_text()
  110. return html.unescape(text)
  111. def getUserUrl(wfRequest: {}) -> str:
  112. if wfRequest.get('links'):
  113. for link in wfRequest['links']:
  114. if link.get('type') and link.get('href'):
  115. if link['type'] == 'application/activity+json':
  116. if not ('/users/' in link['href'] or \
  117. '/profile/' in link['href'] or \
  118. '/channel/' in link['href']):
  119. print('Webfinger activity+json contains single user instance actor')
  120. return link['href']
  121. return None
  122. def parseUserFeed(session,feedUrl: str,asHeader: {}, \
  123. projectVersion: str,httpPrefix: str,domain: str) -> None:
  124. feedJson = getJson(session,feedUrl,asHeader,None, \
  125. projectVersion,httpPrefix,domain)
  126. if not feedJson:
  127. return
  128. if 'orderedItems' in feedJson:
  129. for item in feedJson['orderedItems']:
  130. yield item
  131. nextUrl = None
  132. if 'first' in feedJson:
  133. nextUrl = feedJson['first']
  134. elif 'next' in feedJson:
  135. nextUrl = feedJson['next']
  136. if nextUrl:
  137. if isinstance(nextUrl, str):
  138. userFeed=parseUserFeed(session,nextUrl,asHeader, \
  139. projectVersion,httpPrefix,domain)
  140. for item in userFeed:
  141. yield item
  142. elif isinstance(nextUrl, dict):
  143. userFeed=nextUrl
  144. if userFeed.get('orderedItems'):
  145. for item in userFeed['orderedItems']:
  146. yield item
  147. def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
  148. projectVersion: str,httpPrefix: str, \
  149. nickname: str,domain: str, \
  150. boxName='inbox') -> (str,str,str,str,str,str,str,str):
  151. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  152. if not wfRequest.get('errors'):
  153. personUrl = getUserUrl(wfRequest)
  154. else:
  155. if nickname=='dev':
  156. # try single user instance
  157. print('getPersonBox: Trying single user instance with ld+json')
  158. personUrl = httpPrefix+'://'+domain
  159. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  160. else:
  161. personUrl = httpPrefix+'://'+domain+'/users/'+nickname
  162. if not personUrl:
  163. return None,None,None,None,None,None,None,None
  164. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  165. if not personJson:
  166. if '/channel/' in personUrl:
  167. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  168. personJson = getJson(session,personUrl,asHeader,None, \
  169. projectVersion,httpPrefix,domain)
  170. if not personJson:
  171. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  172. personJson = getJson(session,personUrl,asHeader,None, \
  173. projectVersion,httpPrefix,domain)
  174. if not personJson:
  175. print('Unable to get actor')
  176. return None,None,None,None,None,None,None,None
  177. boxJson=None
  178. if not personJson.get(boxName):
  179. if personJson.get('endpoints'):
  180. if personJson['endpoints'].get(boxName):
  181. boxJson=personJson['endpoints'][boxName]
  182. else:
  183. boxJson=personJson[boxName]
  184. if not boxJson:
  185. return None,None,None,None,None,None,None,None
  186. personId=None
  187. if personJson.get('id'):
  188. personId=personJson['id']
  189. pubKeyId=None
  190. pubKey=None
  191. if personJson.get('publicKey'):
  192. if personJson['publicKey'].get('id'):
  193. pubKeyId=personJson['publicKey']['id']
  194. if personJson['publicKey'].get('publicKeyPem'):
  195. pubKey=personJson['publicKey']['publicKeyPem']
  196. sharedInbox=None
  197. if personJson.get('sharedInbox'):
  198. sharedInbox=personJson['sharedInbox']
  199. else:
  200. if personJson.get('endpoints'):
  201. if personJson['endpoints'].get('sharedInbox'):
  202. sharedInbox=personJson['endpoints']['sharedInbox']
  203. capabilityAcquisition=None
  204. if personJson.get('capabilityAcquisitionEndpoint'):
  205. capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
  206. avatarUrl=None
  207. if personJson.get('icon'):
  208. if personJson['icon'].get('url'):
  209. avatarUrl=personJson['icon']['url']
  210. displayName=None
  211. if personJson.get('name'):
  212. displayName=personJson['name']
  213. storePersonInCache(baseDir,personUrl,personJson,personCache)
  214. return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
  215. def getPosts(session,outboxUrl: str,maxPosts: int, \
  216. maxMentions: int, \
  217. maxEmoji: int,maxAttachments: int, \
  218. federationList: [], \
  219. personCache: {},raw: bool, \
  220. simple: bool,debug: bool, \
  221. projectVersion: str,httpPrefix: str,domain: str) -> {}:
  222. """Gets public posts from an outbox
  223. """
  224. personPosts={}
  225. if not outboxUrl:
  226. return personPosts
  227. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  228. if '/outbox/' in outboxUrl:
  229. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  230. if raw:
  231. result = []
  232. i = 0
  233. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  234. projectVersion,httpPrefix,domain)
  235. for item in userFeed:
  236. result.append(item)
  237. i += 1
  238. if i == maxPosts:
  239. break
  240. pprint(result)
  241. return None
  242. i = 0
  243. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  244. projectVersion,httpPrefix,domain)
  245. for item in userFeed:
  246. if not item.get('id'):
  247. if debug:
  248. print('No id')
  249. continue
  250. if not item.get('type'):
  251. if debug:
  252. print('No type')
  253. continue
  254. if item['type'] != 'Create':
  255. if debug:
  256. print('Not Create type')
  257. continue
  258. if not item.get('object'):
  259. if debug:
  260. print('No object')
  261. continue
  262. if not isinstance(item['object'], dict):
  263. if debug:
  264. print('item object is not a dict')
  265. continue
  266. if not item['object'].get('published'):
  267. if debug:
  268. print('No published attribute')
  269. continue
  270. #pprint(item)
  271. published = item['object']['published']
  272. if not personPosts.get(item['id']):
  273. # check that this is a public post
  274. # #Public should appear in the "to" list
  275. if item['object'].get('to'):
  276. isPublic=False
  277. for recipient in item['object']['to']:
  278. if recipient.endswith('#Public'):
  279. isPublic=True
  280. break
  281. if not isPublic:
  282. continue
  283. content = item['object']['content'].replace('&apos;',"'")
  284. mentions=[]
  285. emoji={}
  286. if item['object'].get('tag'):
  287. for tagItem in item['object']['tag']:
  288. tagType=tagItem['type'].lower()
  289. if tagType=='emoji':
  290. if tagItem.get('name') and tagItem.get('icon'):
  291. if tagItem['icon'].get('url'):
  292. # No emoji from non-permitted domains
  293. if urlPermitted(tagItem['icon']['url'], \
  294. federationList, \
  295. "objects:read"):
  296. emojiName=tagItem['name']
  297. emojiIcon=tagItem['icon']['url']
  298. emoji[emojiName]=emojiIcon
  299. else:
  300. if debug:
  301. print('url not permitted '+tagItem['icon']['url'])
  302. if tagType=='mention':
  303. if tagItem.get('name'):
  304. if tagItem['name'] not in mentions:
  305. mentions.append(tagItem['name'])
  306. if len(mentions)>maxMentions:
  307. if debug:
  308. print('max mentions reached')
  309. continue
  310. if len(emoji)>maxEmoji:
  311. if debug:
  312. print('max emojis reached')
  313. continue
  314. summary = ''
  315. if item['object'].get('summary'):
  316. if item['object']['summary']:
  317. summary = item['object']['summary']
  318. inReplyTo = ''
  319. if item['object'].get('inReplyTo'):
  320. if item['object']['inReplyTo']:
  321. # No replies to non-permitted domains
  322. if not urlPermitted(item['object']['inReplyTo'], \
  323. federationList, \
  324. "objects:read"):
  325. if debug:
  326. print('url not permitted '+item['object']['inReplyTo'])
  327. continue
  328. inReplyTo = item['object']['inReplyTo']
  329. conversation = ''
  330. if item['object'].get('conversation'):
  331. if item['object']['conversation']:
  332. # no conversations originated in non-permitted domains
  333. if urlPermitted(item['object']['conversation'], \
  334. federationList,"objects:read"):
  335. conversation = item['object']['conversation']
  336. attachment = []
  337. if item['object'].get('attachment'):
  338. if item['object']['attachment']:
  339. for attach in item['object']['attachment']:
  340. if attach.get('name') and attach.get('url'):
  341. # no attachments from non-permitted domains
  342. if urlPermitted(attach['url'], \
  343. federationList, \
  344. "objects:read"):
  345. attachment.append([attach['name'],attach['url']])
  346. else:
  347. if debug:
  348. print('url not permitted '+attach['url'])
  349. sensitive = False
  350. if item['object'].get('sensitive'):
  351. sensitive = item['object']['sensitive']
  352. if simple:
  353. print(cleanHtml(content)+'\n')
  354. else:
  355. pprint(item)
  356. personPosts[item['id']] = {
  357. "sensitive": sensitive,
  358. "inreplyto": inReplyTo,
  359. "summary": summary,
  360. "html": content,
  361. "plaintext": cleanHtml(content),
  362. "attachment": attachment,
  363. "mentions": mentions,
  364. "emoji": emoji,
  365. "conversation": conversation
  366. }
  367. i += 1
  368. if i == maxPosts:
  369. break
  370. return personPosts
  371. def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
  372. """Deletes all posts for a person from inbox or outbox
  373. """
  374. if boxname!='inbox' and boxname!='outbox':
  375. return
  376. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  377. for deleteFilename in os.scandir(boxDir):
  378. deleteFilename=deleteFilename.name
  379. filePath = os.path.join(boxDir, deleteFilename)
  380. try:
  381. if os.path.isfile(filePath):
  382. os.unlink(filePath)
  383. elif os.path.isdir(filePath): shutil.rmtree(filePath)
  384. except Exception as e:
  385. print(e)
  386. def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
  387. nickname: str, domain: str,postJsonObject: {}, \
  388. boxname: str) -> str:
  389. """Saves the give json to the give box
  390. Returns the filename
  391. """
  392. if boxname!='inbox' and boxname!='outbox':
  393. return None
  394. originalDomain=domain
  395. if ':' in domain:
  396. domain=domain.split(':')[0]
  397. if not postId:
  398. statusNumber,published = getStatusNumber()
  399. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  400. postJsonObject['id']=postId+'/activity'
  401. if postJsonObject.get('object'):
  402. if isinstance(postJsonObject['object'], dict):
  403. postJsonObject['object']['id']=postId
  404. postJsonObject['object']['atomUri']=postId
  405. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  406. filename=boxDir+'/'+postId.replace('/','#')+'.json'
  407. saveJson(postJsonObject,filename)
  408. return filename
  409. def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
  410. """Writes the post url for hashtags to a file
  411. This allows posts for a hashtag to be quickly looked up
  412. """
  413. if tag['type']!='Hashtag':
  414. return
  415. # create hashtags directory
  416. tagsDir=baseDir+'/tags'
  417. if not os.path.isdir(tagsDir):
  418. os.mkdir(tagsDir)
  419. tagName=tag['name']
  420. tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
  421. tagFile=open(tagsFilename, "a+")
  422. if not tagFile:
  423. return
  424. tagFile.write(newPostId+'\n')
  425. tagFile.close()
  426. def createPostBase(baseDir: str,nickname: str,domain: str,port: int, \
  427. toUrl: str,ccUrl: str,httpPrefix: str,content: str, \
  428. followersOnly: bool,saveToFile: bool,clientToServer: bool, \
  429. attachImageFilename: str,mediaType: str,imageDescription: str, \
  430. useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
  431. inReplyToAtomUri=None,subject=None, \
  432. eventDate=None,eventTime=None,location=None) -> {}:
  433. """Creates a message
  434. """
  435. mentionedRecipients= \
  436. getMentionedPeople(baseDir,httpPrefix,content,domain,False)
  437. tags=[]
  438. hashtagsDict={}
  439. if port:
  440. if port!=80 and port!=443:
  441. if ':' not in domain:
  442. domain=domain+':'+str(port)
  443. # convert content to html
  444. emojisDict={}
  445. # replace 's
  446. content=content.replace("\ufffd\ufffd\ufffds","'s").replace("’","'")
  447. # add tags
  448. content= \
  449. addHtmlTags(baseDir,httpPrefix, \
  450. nickname,domain,content, \
  451. mentionedRecipients, \
  452. hashtagsDict,True)
  453. statusNumber,published = getStatusNumber()
  454. postTo='https://www.w3.org/ns/activitystreams#Public'
  455. postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  456. if followersOnly:
  457. postTo=postCC
  458. postCC=''
  459. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  460. sensitive=False
  461. summary=None
  462. if subject:
  463. summary=subject
  464. sensitive=True
  465. toRecipients=[]
  466. toCC=[]
  467. if toUrl:
  468. if not isinstance(toUrl, str):
  469. print('ERROR: toUrl is not a string')
  470. return None
  471. toRecipients=[toUrl]
  472. # who to send to
  473. if mentionedRecipients:
  474. for mention in mentionedRecipients:
  475. if mention not in toCC:
  476. toCC.append(mention)
  477. # create a list of hashtags
  478. # Only posts which are #Public are searchable by hashtag
  479. if hashtagsDict:
  480. isPublic=False
  481. for recipient in toRecipients:
  482. if recipient.endswith('#Public'):
  483. isPublic=True
  484. break
  485. for tagName,tag in hashtagsDict.items():
  486. tags.append(tag)
  487. if isPublic:
  488. updateHashtagsIndex(baseDir,tag,newPostId)
  489. print('Content tags: '+str(tags))
  490. if inReplyTo and not sensitive:
  491. # locate the post which this is a reply to and check if
  492. # it has a content warning. If it does then reproduce
  493. # the same warning
  494. replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo)
  495. if replyPostFilename:
  496. replyToJson=loadJson(replyPostFilename)
  497. if replyToJson:
  498. if replyToJson.get('object'):
  499. if replyToJson['object'].get('sensitive'):
  500. if replyToJson['object']['sensitive']:
  501. sensitive=True
  502. if replyToJson['object'].get('summary'):
  503. summary=replyToJson['object']['summary']
  504. if eventDate:
  505. eventName=summary
  506. if not eventName:
  507. eventName=content
  508. eventDateStr=eventDate
  509. if eventTime:
  510. if eventTime.endswith('Z'):
  511. eventDateStr=eventDate+'T'+eventTime
  512. else:
  513. eventDateStr=eventDate+'T'+eventTime+':00'+strftime("%z", gmtime())
  514. else:
  515. eventDateStr=eventDate+'T12:00:00Z'
  516. tags.append({
  517. "@context": "https://www.w3.org/ns/activitystreams",
  518. "type": "Event",
  519. "name": eventName,
  520. "startTime": eventDateStr,
  521. "endTime": eventDateStr
  522. })
  523. if location:
  524. tags.append({
  525. "@context": "https://www.w3.org/ns/activitystreams",
  526. "type": "Place",
  527. "name": location
  528. })
  529. postContext=[
  530. 'https://www.w3.org/ns/activitystreams',
  531. {
  532. 'Hashtag': 'as:Hashtag',
  533. 'sensitive': 'as:sensitive',
  534. 'toot': 'http://joinmastodon.org/ns#',
  535. 'votersCount': 'toot:votersCount'
  536. }
  537. ]
  538. if not clientToServer:
  539. actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
  540. # if capabilities have been granted for this actor
  541. # then get the corresponding id
  542. capabilityId=None
  543. capabilityIdList=[]
  544. ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
  545. if ocapFilename:
  546. if os.path.isfile(ocapFilename):
  547. oc=loadJson(ocapFilename)
  548. if oc:
  549. if oc.get('id'):
  550. capabilityIdList=[oc['id']]
  551. newPost = {
  552. "@context": postContext,
  553. 'id': newPostId+'/activity',
  554. 'capability': capabilityIdList,
  555. 'type': 'Create',
  556. 'actor': actorUrl,
  557. 'published': published,
  558. 'to': toRecipients,
  559. 'cc': toCC,
  560. 'object': {
  561. 'id': newPostId,
  562. 'type': 'Note',
  563. 'summary': summary,
  564. 'inReplyTo': inReplyTo,
  565. 'published': published,
  566. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  567. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  568. 'to': toRecipients,
  569. 'cc': toCC,
  570. 'sensitive': sensitive,
  571. 'atomUri': newPostId,
  572. 'inReplyToAtomUri': inReplyToAtomUri,
  573. 'content': content,
  574. 'contentMap': {
  575. 'en': content
  576. },
  577. 'attachment': [],
  578. 'tag': tags,
  579. 'replies': {
  580. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  581. 'type': 'Collection',
  582. 'first': {
  583. 'type': 'CollectionPage',
  584. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  585. 'items': []
  586. }
  587. }
  588. }
  589. }
  590. if attachImageFilename:
  591. newPost['object']= \
  592. attachMedia(baseDir,httpPrefix,domain,port, \
  593. newPost['object'],attachImageFilename, \
  594. mediaType,imageDescription,useBlurhash)
  595. else:
  596. newPost = {
  597. "@context": postContext,
  598. 'id': newPostId,
  599. 'type': 'Note',
  600. 'summary': summary,
  601. 'inReplyTo': inReplyTo,
  602. 'published': published,
  603. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  604. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  605. 'to': toRecipients,
  606. 'cc': toCC,
  607. 'sensitive': sensitive,
  608. 'atomUri': newPostId,
  609. 'inReplyToAtomUri': inReplyToAtomUri,
  610. 'content': content,
  611. 'contentMap': {
  612. 'en': content
  613. },
  614. 'attachment': [],
  615. 'tag': tags,
  616. 'replies': {
  617. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  618. 'type': 'Collection',
  619. 'first': {
  620. 'type': 'CollectionPage',
  621. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  622. 'items': []
  623. }
  624. }
  625. }
  626. if attachImageFilename:
  627. newPost= \
  628. attachMedia(baseDir,httpPrefix,domain,port, \
  629. newPost,attachImageFilename, \
  630. mediaType,imageDescription,useBlurhash)
  631. if ccUrl:
  632. if len(ccUrl)>0:
  633. newPost['cc']=[ccUrl]
  634. if newPost.get('object'):
  635. newPost['object']['cc']=[ccUrl]
  636. # if this is a moderation report then add a status
  637. if isModerationReport:
  638. # add status
  639. if newPost.get('object'):
  640. newPost['object']['moderationStatus']='pending'
  641. else:
  642. newPost['moderationStatus']='pending'
  643. # save to index file
  644. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  645. modFile=open(moderationIndexFile, "a+")
  646. if modFile:
  647. modFile.write(newPostId+'\n')
  648. modFile.close()
  649. if saveToFile:
  650. savePostToBox(baseDir,httpPrefix,newPostId, \
  651. nickname,domain,newPost,'outbox')
  652. return newPost
  653. def outboxMessageCreateWrap(httpPrefix: str, \
  654. nickname: str,domain: str,port: int, \
  655. messageJson: {}) -> {}:
  656. """Wraps a received message in a Create
  657. https://www.w3.org/TR/activitypub/#object-without-create
  658. """
  659. if port:
  660. if port!=80 and port!=443:
  661. if ':' not in domain:
  662. domain=domain+':'+str(port)
  663. statusNumber,published = getStatusNumber()
  664. if messageJson.get('published'):
  665. published = messageJson['published']
  666. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  667. cc=[]
  668. if messageJson.get('cc'):
  669. cc=messageJson['cc']
  670. # TODO
  671. capabilityUrl=[]
  672. newPost = {
  673. "@context": "https://www.w3.org/ns/activitystreams",
  674. 'id': newPostId+'/activity',
  675. 'capability': capabilityUrl,
  676. 'type': 'Create',
  677. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  678. 'published': published,
  679. 'to': messageJson['to'],
  680. 'cc': cc,
  681. 'object': messageJson
  682. }
  683. newPost['object']['id']=newPost['id']
  684. newPost['object']['url']= \
  685. httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
  686. newPost['object']['atomUri']= \
  687. httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  688. return newPost
  689. def postIsAddressedToFollowers(baseDir: str,
  690. nickname: str,domain: str,port: int,httpPrefix: str,
  691. postJsonObject: {}) -> bool:
  692. """Returns true if the given post is addressed to followers of the nickname
  693. """
  694. if port:
  695. if port!=80 and port!=443:
  696. if ':' not in domain:
  697. domain=domain+':'+str(port)
  698. if not postJsonObject.get('object'):
  699. return False
  700. toList=[]
  701. ccList=[]
  702. if postJsonObject['type']!='Update' and \
  703. isinstance(postJsonObject['object'], dict):
  704. if postJsonObject['object'].get('to'):
  705. toList=postJsonObject['object']['to']
  706. if postJsonObject['object'].get('cc'):
  707. ccList=postJsonObject['object']['cc']
  708. else:
  709. if postJsonObject.get('to'):
  710. toList=postJsonObject['to']
  711. if postJsonObject.get('cc'):
  712. ccList=postJsonObject['cc']
  713. followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  714. # does the followers url exist in 'to' or 'cc' lists?
  715. addressedToFollowers=False
  716. if followersUrl in toList:
  717. addressedToFollowers=True
  718. elif followersUrl in ccList:
  719. addressedToFollowers=True
  720. return addressedToFollowers
  721. def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
  722. """Returns true if the given post is addressed to public
  723. """
  724. if not postJsonObject.get('object'):
  725. return False
  726. if not postJsonObject['object'].get('to'):
  727. return False
  728. publicUrl='https://www.w3.org/ns/activitystreams#Public'
  729. # does the public url exist in 'to' or 'cc' lists?
  730. addressedToPublic=False
  731. if publicUrl in postJsonObject['object']['to']:
  732. addressedToPublic=True
  733. if not addressedToPublic:
  734. if not postJsonObject['object'].get('cc'):
  735. return False
  736. if publicUrl in postJsonObject['object']['cc']:
  737. addressedToPublic=True
  738. return addressedToPublic
  739. def createPublicPost(baseDir: str,
  740. nickname: str,domain: str,port: int,httpPrefix: str, \
  741. content: str,followersOnly: bool,saveToFile: bool,
  742. clientToServer: bool,\
  743. attachImageFilename: str,mediaType: str, \
  744. imageDescription: str,useBlurhash: bool, \
  745. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  746. eventDate=None,eventTime=None,location=None) -> {}:
  747. """Public post
  748. """
  749. domainFull=domain
  750. if port:
  751. if port!=80 and port!=443:
  752. if ':' not in domain:
  753. domainFull=domain+':'+str(port)
  754. return createPostBase(baseDir,nickname,domain,port, \
  755. 'https://www.w3.org/ns/activitystreams#Public', \
  756. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  757. httpPrefix,content,followersOnly,saveToFile, \
  758. clientToServer, \
  759. attachImageFilename,mediaType, \
  760. imageDescription,useBlurhash, \
  761. False,inReplyTo,inReplyToAtomUri,subject, \
  762. eventDate,eventTime,location)
  763. def createQuestionPost(baseDir: str,
  764. nickname: str,domain: str,port: int,httpPrefix: str, \
  765. content: str,qOptions: [], \
  766. followersOnly: bool,saveToFile: bool,
  767. clientToServer: bool,\
  768. attachImageFilename: str,mediaType: str, \
  769. imageDescription: str,useBlurhash: bool, \
  770. subject: str,durationDays: int) -> {}:
  771. """Question post with multiple choice options
  772. """
  773. domainFull=domain
  774. if port:
  775. if port!=80 and port!=443:
  776. if ':' not in domain:
  777. domainFull=domain+':'+str(port)
  778. messageJson= \
  779. createPostBase(baseDir,nickname,domain,port, \
  780. 'https://www.w3.org/ns/activitystreams#Public', \
  781. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  782. httpPrefix,content,followersOnly,saveToFile, \
  783. clientToServer, \
  784. attachImageFilename,mediaType, \
  785. imageDescription,useBlurhash, \
  786. False,None,None,subject, \
  787. None,None,None)
  788. messageJson['object']['type']='Question'
  789. messageJson['object']['oneOf']=[]
  790. messageJson['object']['votersCount']=0
  791. currTime=datetime.datetime.utcnow()
  792. daysSinceEpoch=int((currTime - datetime.datetime(1970,1,1)).days + durationDays)
  793. endTime=datetime.datetime(1970,1,1) + datetime.timedelta(daysSinceEpoch)
  794. messageJson['object']['endTime']=endTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  795. for questionOption in qOptions:
  796. messageJson['object']['oneOf'].append({
  797. "type": "Note",
  798. "name": questionOption,
  799. "replies": {
  800. "type": "Collection",
  801. "totalItems": 0
  802. }
  803. })
  804. return messageJson
  805. def createUnlistedPost(baseDir: str,
  806. nickname: str,domain: str,port: int,httpPrefix: str, \
  807. content: str,followersOnly: bool,saveToFile: bool,
  808. clientToServer: bool,\
  809. attachImageFilename: str,mediaType: str, \
  810. imageDescription: str,useBlurhash: bool, \
  811. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  812. eventDate=None,eventTime=None,location=None) -> {}:
  813. """Unlisted post. This has the #Public and followers links inverted.
  814. """
  815. domainFull=domain
  816. if port:
  817. if port!=80 and port!=443:
  818. if ':' not in domain:
  819. domainFull=domain+':'+str(port)
  820. return createPostBase(baseDir,nickname,domain,port, \
  821. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  822. 'https://www.w3.org/ns/activitystreams#Public', \
  823. httpPrefix,content,followersOnly,saveToFile, \
  824. clientToServer, \
  825. attachImageFilename,mediaType, \
  826. imageDescription,useBlurhash, \
  827. False,inReplyTo, inReplyToAtomUri, subject, \
  828. eventDate,eventTime,location)
  829. def createFollowersOnlyPost(baseDir: str,
  830. nickname: str,domain: str,port: int,httpPrefix: str, \
  831. content: str,followersOnly: bool,saveToFile: bool,
  832. clientToServer: bool,\
  833. attachImageFilename: str,mediaType: str, \
  834. imageDescription: str,useBlurhash: bool, \
  835. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  836. eventDate=None,eventTime=None,location=None) -> {}:
  837. """Followers only post
  838. """
  839. domainFull=domain
  840. if port:
  841. if port!=80 and port!=443:
  842. if ':' not in domain:
  843. domainFull=domain+':'+str(port)
  844. return createPostBase(baseDir,nickname,domain,port, \
  845. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  846. None,
  847. httpPrefix,content,followersOnly,saveToFile, \
  848. clientToServer, \
  849. attachImageFilename,mediaType, \
  850. imageDescription,useBlurhash, \
  851. False,inReplyTo, inReplyToAtomUri, subject, \
  852. eventDate,eventTime,location)
  853. def getMentionedPeople(baseDir: str,httpPrefix: str, \
  854. content: str,domain: str,debug: bool) -> []:
  855. """Extracts a list of mentioned actors from the given message content
  856. """
  857. if '@' not in content:
  858. return None
  859. mentions=[]
  860. words=content.split(' ')
  861. for wrd in words:
  862. if wrd.startswith('@'):
  863. handle=wrd[1:]
  864. if debug:
  865. print('DEBUG: mentioned handle '+handle)
  866. if '@' not in handle:
  867. handle=handle+'@'+domain
  868. if not os.path.isdir(baseDir+'/accounts/'+handle):
  869. continue
  870. else:
  871. externalDomain=handle.split('@')[1]
  872. if not ('.' in externalDomain or externalDomain=='localhost'):
  873. continue
  874. mentionedNickname=handle.split('@')[0]
  875. mentionedDomain=handle.split('@')[1].strip('\n')
  876. if ':' in mentionedDomain:
  877. mentionedDomain=mentionedDomain.split(':')[0]
  878. if not validNickname(mentionedDomain,mentionedNickname):
  879. continue
  880. actor=httpPrefix+'://'+handle.split('@')[1]+'/users/'+mentionedNickname
  881. mentions.append(actor)
  882. return mentions
  883. def createDirectMessagePost(baseDir: str,
  884. nickname: str,domain: str,port: int,httpPrefix: str, \
  885. content: str,followersOnly: bool,saveToFile: bool,
  886. clientToServer: bool,\
  887. attachImageFilename: str,mediaType: str, \
  888. imageDescription: str,useBlurhash: bool, \
  889. inReplyTo=None,inReplyToAtomUri=None,subject=None,debug=False, \
  890. eventDate=None,eventTime=None,location=None) -> {}:
  891. """Direct Message post
  892. """
  893. mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
  894. if debug:
  895. print('mentionedPeople: '+str(mentionedPeople))
  896. if not mentionedPeople:
  897. return None
  898. postTo=None
  899. postCc=None
  900. messageJson= \
  901. createPostBase(baseDir,nickname,domain,port, \
  902. postTo,postCc, \
  903. httpPrefix,content,followersOnly,saveToFile, \
  904. clientToServer, \
  905. attachImageFilename,mediaType, \
  906. imageDescription,useBlurhash, \
  907. False,inReplyTo,inReplyToAtomUri,subject, \
  908. eventDate,eventTime,location)
  909. # mentioned recipients go into To rather than Cc
  910. messageJson['to']=messageJson['object']['cc']
  911. messageJson['object']['to']=messageJson['to']
  912. messageJson['cc']=[]
  913. messageJson['object']['cc']=[]
  914. return messageJson
  915. def createReportPost(baseDir: str,
  916. nickname: str, domain: str, port: int,httpPrefix: str, \
  917. content: str, followersOnly: bool, saveToFile: bool,
  918. clientToServer: bool,\
  919. attachImageFilename: str,mediaType: str, \
  920. imageDescription: str,useBlurhash: bool, \
  921. debug: bool,subject=None) -> {}:
  922. """Send a report to moderators
  923. """
  924. domainFull=domain
  925. if port:
  926. if port!=80 and port!=443:
  927. if ':' not in domain:
  928. domainFull=domain+':'+str(port)
  929. # add a title to distinguish moderation reports from other posts
  930. reportTitle='Moderation Report'
  931. if not subject:
  932. subject=reportTitle
  933. else:
  934. if not subject.startswith(reportTitle):
  935. subject=reportTitle+': '+subject
  936. # create the list of moderators from the moderators file
  937. moderatorsList=[]
  938. moderatorsFile=baseDir+'/accounts/moderators.txt'
  939. if os.path.isfile(moderatorsFile):
  940. with open (moderatorsFile, "r") as fileHandler:
  941. for line in fileHandler:
  942. line=line.strip('\n')
  943. if line.startswith('#'):
  944. continue
  945. if line.startswith('/users/'):
  946. line=line.replace('users','')
  947. if line.startswith('@'):
  948. line=line[1:]
  949. if '@' in line:
  950. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
  951. if moderatorActor not in moderatorList:
  952. moderatorsList.append(moderatorActor)
  953. continue
  954. if line.startswith('http') or line.startswith('dat'):
  955. # must be a local address - no remote moderators
  956. if '://'+domainFull+'/' in line:
  957. if line not in moderatorsList:
  958. moderatorsList.append(line)
  959. else:
  960. if '/' not in line:
  961. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
  962. if moderatorActor not in moderatorsList:
  963. moderatorsList.append(moderatorActor)
  964. if len(moderatorsList)==0:
  965. # if there are no moderators then the admin becomes the moderator
  966. adminNickname=getConfigParam(baseDir,'admin')
  967. if adminNickname:
  968. moderatorsList.append(httpPrefix+'://'+domainFull+'/users/'+adminNickname)
  969. if not moderatorsList:
  970. return None
  971. if debug:
  972. print('DEBUG: Sending report to moderators')
  973. print(str(moderatorsList))
  974. postTo=moderatorsList
  975. postCc=None
  976. postJsonObject=None
  977. for toUrl in postTo:
  978. # who is this report going to?
  979. toNickname=toUrl.split('/users/')[1]
  980. handle=toNickname+'@'+domain
  981. postJsonObject= \
  982. createPostBase(baseDir,nickname,domain,port, \
  983. toUrl,postCc, \
  984. httpPrefix,content,followersOnly,saveToFile, \
  985. clientToServer, \
  986. attachImageFilename,mediaType, \
  987. imageDescription,useBlurhash, \
  988. True,None,None,subject, \
  989. None,None,None)
  990. if not postJsonObject:
  991. continue
  992. # update the inbox index with the report filename
  993. #indexFilename=baseDir+'/accounts/'+handle+'/inbox.index'
  994. #indexEntry=postJsonObject['id'].replace('/activity','').replace('/','#')+'.json'
  995. #if indexEntry not in open(indexFilename).read():
  996. # try:
  997. # with open(indexFilename, 'a+') as fp:
  998. # fp.write(indexEntry)
  999. # except:
  1000. # pass
  1001. # save a notification file so that the moderator
  1002. # knows something new has appeared
  1003. newReportFile=baseDir+'/accounts/'+handle+'/.newReport'
  1004. if os.path.isfile(newReportFile):
  1005. continue
  1006. try:
  1007. with open(newReportFile, 'w') as fp:
  1008. fp.write(toUrl+'/moderation')
  1009. except:
  1010. pass
  1011. return postJsonObject
  1012. def threadSendPost(session,postJsonStr: str,federationList: [],\
  1013. inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
  1014. debug :bool) -> None:
  1015. """Sends a with retries
  1016. """
  1017. tries=0
  1018. sendIntervalSec=30
  1019. for attempt in range(20):
  1020. postResult=None
  1021. unauthorized=False
  1022. try:
  1023. postResult,unauthorized = \
  1024. postJsonString(session,postJsonStr,federationList, \
  1025. inboxUrl,signatureHeaderJson, \
  1026. "inbox:write",debug)
  1027. except Exception as e:
  1028. print('ERROR: postJsonString failed '+str(e))
  1029. if unauthorized==True:
  1030. print(postJsonStr)
  1031. print('threadSendPost: Post is unauthorized')
  1032. break
  1033. if postResult:
  1034. logStr='Success on try '+str(tries)+': '+postJsonStr
  1035. else:
  1036. logStr='Retry '+str(tries)+': '+postJsonStr
  1037. postLog.append(logStr)
  1038. # keep the length of the log finite
  1039. # Don't accumulate massive files on systems with limited resources
  1040. while len(postLog)>16:
  1041. postLog.pop(0)
  1042. if debug:
  1043. # save the log file
  1044. postLogFilename=baseDir+'/post.log'
  1045. with open(postLogFilename, "a+") as logFile:
  1046. logFile.write(logStr+'\n')
  1047. if postResult:
  1048. if debug:
  1049. print('DEBUG: successful json post to '+inboxUrl)
  1050. # our work here is done
  1051. break
  1052. if debug:
  1053. print(postJsonStr)
  1054. print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
  1055. str(sendIntervalSec)+' seconds.')
  1056. time.sleep(sendIntervalSec)
  1057. tries+=1
  1058. def sendPost(projectVersion: str, \
  1059. session,baseDir: str,nickname: str, domain: str, port: int, \
  1060. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1061. httpPrefix: str, content: str, followersOnly: bool, \
  1062. saveToFile: bool, clientToServer: bool, \
  1063. attachImageFilename: str,mediaType: str, \
  1064. imageDescription: str,useBlurhash: bool, \
  1065. federationList: [],\
  1066. sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
  1067. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  1068. """Post to another inbox
  1069. """
  1070. withDigest=True
  1071. if toNickname=='inbox':
  1072. # shared inbox actor on @domain@domain
  1073. toNickname=toDomain
  1074. toDomainOriginal=toDomain
  1075. if toPort:
  1076. if toPort!=80 and toPort!=443:
  1077. if ':' not in toDomain:
  1078. toDomain=toDomain+':'+str(toPort)
  1079. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  1080. # lookup the inbox for the To handle
  1081. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1082. domain,projectVersion)
  1083. if not wfRequest:
  1084. return 1
  1085. if not clientToServer:
  1086. postToBox='inbox'
  1087. else:
  1088. postToBox='outbox'
  1089. # get the actor inbox for the To handle
  1090. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1091. getPersonBox(baseDir,session,wfRequest,personCache, \
  1092. projectVersion,httpPrefix, \
  1093. nickname,domain,postToBox)
  1094. # If there are more than one followers on the target domain
  1095. # then send to the shared inbox indead of the individual inbox
  1096. if nickname=='capabilities':
  1097. inboxUrl=capabilityAcquisition
  1098. if not capabilityAcquisition:
  1099. return 2
  1100. if not inboxUrl:
  1101. return 3
  1102. if not pubKey:
  1103. return 4
  1104. if not toPersonId:
  1105. return 5
  1106. # sharedInbox and capabilities are optional
  1107. postJsonObject = \
  1108. createPostBase(baseDir,nickname,domain,port, \
  1109. toPersonId,cc,httpPrefix,content, \
  1110. followersOnly,saveToFile,clientToServer, \
  1111. attachImageFilename,mediaType, \
  1112. imageDescription,useBlurhash, \
  1113. False,inReplyTo,inReplyToAtomUri,subject, \
  1114. None,None,None)
  1115. # get the senders private key
  1116. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
  1117. if len(privateKeyPem)==0:
  1118. return 6
  1119. if toDomain not in inboxUrl:
  1120. return 7
  1121. postPath=inboxUrl.split(toDomain,1)[1]
  1122. # convert json to string so that there are no
  1123. # subsequent conversions after creating message body digest
  1124. postJsonStr=json.dumps(postJsonObject)
  1125. # construct the http header, including the message body digest
  1126. signatureHeaderJson = \
  1127. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1128. toDomain,toPort, \
  1129. postPath,httpPrefix,withDigest,postJsonStr)
  1130. # Keep the number of threads being used small
  1131. while len(sendThreads)>1000:
  1132. print('WARN: Maximum threads reached - killing send thread')
  1133. sendThreads[0].kill()
  1134. sendThreads.pop(0)
  1135. print('WARN: thread killed')
  1136. thr = threadWithTrace(target=threadSendPost,args=(session, \
  1137. postJsonStr, \
  1138. federationList, \
  1139. inboxUrl,baseDir, \
  1140. signatureHeaderJson.copy(), \
  1141. postLog,
  1142. debug),daemon=True)
  1143. sendThreads.append(thr)
  1144. thr.start()
  1145. return 0
  1146. def sendPostViaServer(projectVersion: str, \
  1147. baseDir: str,session,fromNickname: str,password: str, \
  1148. fromDomain: str, fromPort: int, \
  1149. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1150. httpPrefix: str, content: str, followersOnly: bool, \
  1151. attachImageFilename: str,mediaType: str, \
  1152. imageDescription: str,useBlurhash: bool, \
  1153. cachedWebfingers: {},personCache: {}, \
  1154. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  1155. """Send a post via a proxy (c2s)
  1156. """
  1157. if not session:
  1158. print('WARN: No session for sendPostViaServer')
  1159. return 6
  1160. withDigest=True
  1161. if toPort:
  1162. if toPort!=80 and toPort!=443:
  1163. if ':' not in fromDomain:
  1164. fromDomain=fromDomain+':'+str(fromPort)
  1165. handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
  1166. # lookup the inbox for the To handle
  1167. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1168. fromDomain,projectVersion)
  1169. if not wfRequest:
  1170. if debug:
  1171. print('DEBUG: webfinger failed for '+handle)
  1172. return 1
  1173. postToBox='outbox'
  1174. # get the actor inbox for the To handle
  1175. inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1176. getPersonBox(baseDir,session,wfRequest,personCache, \
  1177. projectVersion,httpPrefix,fromNickname, \
  1178. fromDomain,postToBox)
  1179. if not inboxUrl:
  1180. if debug:
  1181. print('DEBUG: No '+postToBox+' was found for '+handle)
  1182. return 3
  1183. if not fromPersonId:
  1184. if debug:
  1185. print('DEBUG: No actor was found for '+handle)
  1186. return 4
  1187. # Get the json for the c2s post, not saving anything to file
  1188. # Note that baseDir is set to None
  1189. saveToFile=False
  1190. clientToServer=True
  1191. if toDomain.lower().endswith('public'):
  1192. toPersonId='https://www.w3.org/ns/activitystreams#Public'
  1193. fromDomainFull=fromDomain
  1194. if fromPort:
  1195. if fromPort!=80 and fromPort!=443:
  1196. if ':' not in fromDomain:
  1197. fromDomainFull=fromDomain+':'+str(fromPort)
  1198. cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1199. else:
  1200. if toDomain.lower().endswith('followers') or \
  1201. toDomain.lower().endswith('followersonly'):
  1202. toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1203. else:
  1204. toDomainFull=toDomain
  1205. if toPort:
  1206. if toPort!=80 and toPort!=443:
  1207. if ':' not in toDomain:
  1208. toDomainFull=toDomain+':'+str(toPort)
  1209. toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
  1210. postJsonObject = \
  1211. createPostBase(baseDir, \
  1212. fromNickname,fromDomain,fromPort, \
  1213. toPersonId,cc,httpPrefix,content, \
  1214. followersOnly,saveToFile,clientToServer, \
  1215. attachImageFilename,mediaType, \
  1216. imageDescription,useBlurhash, \
  1217. False,inReplyTo,inReplyToAtomUri,subject, \
  1218. None,None,None)
  1219. authHeader=createBasicAuthHeader(fromNickname,password)
  1220. if attachImageFilename:
  1221. headers = {'host': fromDomain, \
  1222. 'Authorization': authHeader}
  1223. postResult = \
  1224. postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
  1225. #if not postResult:
  1226. # if debug:
  1227. # print('DEBUG: Failed to upload image')
  1228. # return 9
  1229. headers = {'host': fromDomain, \
  1230. 'Content-type': 'application/json', \
  1231. 'Authorization': authHeader}
  1232. postResult = \
  1233. postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write",debug)
  1234. #if not postResult:
  1235. # if debug:
  1236. # print('DEBUG: POST failed for c2s to '+inboxUrl)
  1237. # return 5
  1238. if debug:
  1239. print('DEBUG: c2s POST success')
  1240. return 0
  1241. def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
  1242. """Returns a dictionary with followers grouped by domain
  1243. """
  1244. handle=nickname+'@'+domain
  1245. followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
  1246. if not os.path.isfile(followersFilename):
  1247. return None
  1248. grouped={}
  1249. with open(followersFilename, "r") as f:
  1250. for followerHandle in f:
  1251. if '@' in followerHandle:
  1252. fHandle=followerHandle.strip().replace('\n','')
  1253. followerDomain=fHandle.split('@')[1]
  1254. if not grouped.get(followerDomain):
  1255. grouped[followerDomain]=[fHandle]
  1256. else:
  1257. grouped[followerDomain].append(fHandle)
  1258. return grouped
  1259. def addFollowersToPublicPost(postJsonObject: {}) -> None:
  1260. """Adds followers entry to cc if it doesn't exist
  1261. """
  1262. if not postJsonObject.get('actor'):
  1263. return
  1264. if isinstance(postJsonObject['object'], str):
  1265. if not postJsonObject.get('to'):
  1266. return
  1267. if len(postJsonObject['to'])>1:
  1268. return
  1269. if len(postJsonObject['to'])==0:
  1270. return
  1271. if not postJsonObject['to'][0].endswith('#Public'):
  1272. return
  1273. if postJsonObject.get('cc'):
  1274. return
  1275. postJsonObject['cc']=postJsonObject['actor']+'/followers'
  1276. elif isinstance(postJsonObject['object'], dict):
  1277. if not postJsonObject['object'].get('to'):
  1278. return
  1279. if len(postJsonObject['object']['to'])>1:
  1280. return
  1281. if len(postJsonObject['object']['to'])==0:
  1282. return
  1283. if not postJsonObject['object']['to'][0].endswith('#Public'):
  1284. return
  1285. if postJsonObject['object'].get('cc'):
  1286. return
  1287. postJsonObject['object']['cc']=postJsonObject['actor']+'/followers'
  1288. def sendSignedJson(postJsonObject: {},session,baseDir: str, \
  1289. nickname: str, domain: str, port: int, \
  1290. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1291. httpPrefix: str, saveToFile: bool, clientToServer: bool, \
  1292. federationList: [], \
  1293. sendThreads: [], postLog: [], cachedWebfingers: {}, \
  1294. personCache: {}, debug: bool,projectVersion: str) -> int:
  1295. """Sends a signed json object to an inbox/outbox
  1296. """
  1297. if debug:
  1298. print('DEBUG: sendSignedJson start')
  1299. if not session:
  1300. print('WARN: No session specified for sendSignedJson')
  1301. return 8
  1302. withDigest=True
  1303. sharedInbox=False
  1304. if toNickname=='inbox':
  1305. # shared inbox actor on @domain@domain
  1306. toNickname=toDomain
  1307. sharedInbox=True
  1308. toDomainOriginal=toDomain
  1309. if toPort:
  1310. if toPort!=80 and toPort!=443:
  1311. if ':' not in toDomain:
  1312. toDomain=toDomain+':'+str(toPort)
  1313. handleBase=httpPrefix+'://'+toDomain+'/@'
  1314. if toNickname:
  1315. handle=handleBase+toNickname
  1316. else:
  1317. singleUserInstanceNickname='dev'
  1318. handle=handleBase+singleUserInstanceNickname
  1319. if debug:
  1320. print('DEBUG: handle - '+handle+' toPort '+str(toPort))
  1321. # lookup the inbox for the To handle
  1322. wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1323. domain,projectVersion)
  1324. if not wfRequest:
  1325. if debug:
  1326. print('DEBUG: webfinger for '+handle+' failed')
  1327. return 1
  1328. if wfRequest.get('errors'):
  1329. if debug:
  1330. print('DEBUG: webfinger for '+handle+' failed with errors '+str(wfRequest['errors']))
  1331. if not clientToServer:
  1332. postToBox='inbox'
  1333. else:
  1334. postToBox='outbox'
  1335. # get the actor inbox/outbox/capabilities for the To handle
  1336. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
  1337. getPersonBox(baseDir,session,wfRequest,personCache, \
  1338. projectVersion,httpPrefix,nickname,domain,postToBox)
  1339. if nickname=='capabilities':
  1340. inboxUrl=capabilityAcquisition
  1341. if not capabilityAcquisition:
  1342. return 2
  1343. else:
  1344. print("inboxUrl: "+str(inboxUrl))
  1345. print("toPersonId: "+str(toPersonId))
  1346. print("sharedInboxUrl: "+str(sharedInboxUrl))
  1347. if inboxUrl:
  1348. if inboxUrl.endswith('/actor/inbox'):
  1349. inboxUrl=sharedInboxUrl
  1350. if not inboxUrl:
  1351. if debug:
  1352. print('DEBUG: missing inboxUrl')
  1353. return 3
  1354. if debug:
  1355. print('DEBUG: Sending to endpoint '+inboxUrl)
  1356. if not pubKey:
  1357. if debug:
  1358. print('DEBUG: missing pubkey')
  1359. return 4
  1360. if not toPersonId:
  1361. if debug:
  1362. print('DEBUG: missing personId')
  1363. return 5
  1364. # sharedInbox and capabilities are optional
  1365. # get the senders private key
  1366. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
  1367. if len(privateKeyPem)==0:
  1368. if debug:
  1369. print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
  1370. return 6
  1371. if toDomain not in inboxUrl:
  1372. if debug:
  1373. print('DEBUG: '+toDomain+' is not in '+inboxUrl)
  1374. return 7
  1375. postPath=inboxUrl.split(toDomain,1)[1]
  1376. addFollowersToPublicPost(postJsonObject)
  1377. # convert json to string so that there are no
  1378. # subsequent conversions after creating message body digest
  1379. postJsonStr=json.dumps(postJsonObject)
  1380. # construct the http header, including the message body digest
  1381. signatureHeaderJson = \
  1382. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1383. toDomain,toPort, \
  1384. postPath,httpPrefix,withDigest,postJsonStr)
  1385. # Keep the number of threads being used small
  1386. while len(sendThreads)>1000:
  1387. print('WARN: Maximum threads reached - killing send thread')
  1388. sendThreads[0].kill()
  1389. sendThreads.pop(0)
  1390. print('WARN: thread killed')
  1391. if debug:
  1392. print('DEBUG: starting thread to send post')
  1393. pprint(postJsonObject)
  1394. thr = threadWithTrace(target=threadSendPost, \
  1395. args=(session, \
  1396. postJsonStr, \
  1397. federationList, \
  1398. inboxUrl,baseDir, \
  1399. signatureHeaderJson.copy(), \
  1400. postLog,
  1401. debug),daemon=True)
  1402. sendThreads.append(thr)
  1403. #thr.start()
  1404. return 0
  1405. def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
  1406. """The Follow activity doesn't have a 'to' field and so one
  1407. needs to be added so that activity distribution happens in a consistent way
  1408. Returns true if a 'to' field exists or was added
  1409. """
  1410. if postJsonObject.get('to'):
  1411. return postJsonObject,True
  1412. if debug:
  1413. pprint(postJsonObject)
  1414. print('DEBUG: no "to" field when sending to named addresses 2')
  1415. isSameType=False
  1416. toFieldAdded=False
  1417. if postJsonObject.get('object'):
  1418. if isinstance(postJsonObject['object'], str):
  1419. if postJsonObject.get('type'):
  1420. if postJsonObject['type']==activityType:
  1421. isSameType=True
  1422. if debug:
  1423. print('DEBUG: "to" field assigned to Follow')
  1424. toAddress=postJsonObject['object']
  1425. if '/statuses/' in toAddress:
  1426. toAddress=toAddress.split('/statuses/')[0]
  1427. postJsonObject['to']=[toAddress]
  1428. toFieldAdded=True
  1429. elif isinstance(postJsonObject['object'], dict):
  1430. if postJsonObject['object'].get('type'):
  1431. if postJsonObject['object']['type']==activityType:
  1432. isSameType=True
  1433. if isinstance(postJsonObject['object']['object'], str):
  1434. if debug:
  1435. print('DEBUG: "to" field assigned to Follow')
  1436. toAddress=postJsonObject['object']['object']
  1437. if '/statuses/' in toAddress:
  1438. toAddress=toAddress.split('/statuses/')[0]
  1439. postJsonObject['object']['to']=[toAddress]
  1440. postJsonObject['to']=[postJsonObject['object']['object']]
  1441. toFieldAdded=True
  1442. if not isSameType:
  1443. return postJsonObject,True
  1444. if toFieldAdded:
  1445. return postJsonObject,True
  1446. return postJsonObject,False
  1447. def sendToNamedAddresses(session,baseDir: str, \
  1448. nickname: str, domain: str, port: int, \
  1449. httpPrefix: str,federationList: [], \
  1450. sendThreads: [],postLog: [], \
  1451. cachedWebfingers: {},personCache: {}, \
  1452. postJsonObject: {},debug: bool, \
  1453. projectVersion: str) -> None:
  1454. """sends a post to the specific named addresses in to/cc
  1455. """
  1456. if not session:
  1457. print('WARN: No session for sendToNamedAddresses')
  1458. return
  1459. if not postJsonObject.get('object'):
  1460. return
  1461. if isinstance(postJsonObject['object'], dict):
  1462. isProfileUpdate=False
  1463. # for actor updates there is no 'to' within the object
  1464. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1465. if postJsonObject['type']=='Update' and \
  1466. (postJsonObject['object']['type']=='Person' or \
  1467. postJsonObject['object']['type']=='Application' or \
  1468. postJsonObject['object']['type']=='Group' or \
  1469. postJsonObject['object']['type']=='Service'):
  1470. # use the original object, which has a 'to'
  1471. recipientsObject=postJsonObject
  1472. isProfileUpdate=True
  1473. if not isProfileUpdate:
  1474. if not postJsonObject['object'].get('to'):
  1475. if debug:
  1476. pprint(postJsonObject)
  1477. print('DEBUG: no "to" field when sending to named addresses')
  1478. if postJsonObject['object'].get('type'):
  1479. if postJsonObject['object']['type']=='Follow':
  1480. if isinstance(postJsonObject['object']['object'], str):
  1481. if debug:
  1482. print('DEBUG: "to" field assigned to Follow')
  1483. postJsonObject['object']['to']=[postJsonObject['object']['object']]
  1484. if not postJsonObject['object'].get('to'):
  1485. return
  1486. recipientsObject=postJsonObject['object']
  1487. else:
  1488. postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
  1489. if not fieldAdded:
  1490. return
  1491. postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
  1492. if not fieldAdded:
  1493. return
  1494. recipientsObject=postJsonObject
  1495. recipients=[]
  1496. recipientType=['to','cc']
  1497. for rType in recipientType:
  1498. if not recipientsObject.get(rType):
  1499. continue
  1500. if isinstance(recipientsObject[rType], list):
  1501. if debug:
  1502. pprint(recipientsObject)
  1503. print('recipientsObject: '+str(recipientsObject[rType]))
  1504. for address in recipientsObject[rType]:
  1505. if not address:
  1506. continue
  1507. if '/' not in address:
  1508. continue
  1509. if address.endswith('#Public'):
  1510. continue
  1511. if address.endswith('/followers'):
  1512. continue
  1513. recipients.append(address)
  1514. elif isinstance(recipientsObject[rType], str):
  1515. address=recipientsObject[rType]
  1516. if address:
  1517. if '/' in address:
  1518. if address.endswith('#Public'):
  1519. continue
  1520. if address.endswith('/followers'):
  1521. continue
  1522. recipients.append(address)
  1523. if not recipients:
  1524. if debug:
  1525. print('DEBUG: no individual recipients')
  1526. return
  1527. if debug:
  1528. print('DEBUG: Sending individually addressed posts: '+str(recipients))
  1529. # this is after the message has arrived at the server
  1530. clientToServer=False
  1531. for address in recipients:
  1532. toNickname=getNicknameFromActor(address)
  1533. if not toNickname:
  1534. continue
  1535. toDomain,toPort=getDomainFromActor(address)
  1536. if not toDomain:
  1537. continue
  1538. if debug:
  1539. domainFull=domain
  1540. if port:
  1541. if port!=80 and port!=443:
  1542. if ':' not in domain:
  1543. domainFull=domain+':'+str(port)
  1544. toDomainFull=toDomain
  1545. if toPort:
  1546. if toPort!=80 and toPort!=443:
  1547. if ':' not in toDomain:
  1548. toDomainFull=toDomain+':'+str(toPort)
  1549. print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
  1550. cc=[]
  1551. sendSignedJson(postJsonObject,session,baseDir, \
  1552. nickname,domain,port, \
  1553. toNickname,toDomain,toPort, \
  1554. cc,httpPrefix,True,clientToServer, \
  1555. federationList, \
  1556. sendThreads,postLog,cachedWebfingers, \
  1557. personCache,debug,projectVersion)
  1558. def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
  1559. """Returns true if the given domain has a shared inbox
  1560. """
  1561. wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
  1562. None,__version__)
  1563. if wfRequest:
  1564. if not wfRequest.get('errors'):
  1565. return True
  1566. return False
  1567. def sendToFollowers(session,baseDir: str, \
  1568. nickname: str, domain: str, port: int, \
  1569. httpPrefix: str,federationList: [], \
  1570. sendThreads: [],postLog: [], \
  1571. cachedWebfingers: {},personCache: {}, \
  1572. postJsonObject: {},debug: bool, \
  1573. projectVersion: str) -> None:
  1574. """sends a post to the followers of the given nickname
  1575. """
  1576. print('sendToFollowers')
  1577. if not session:
  1578. print('WARN: No session for sendToFollowers')
  1579. return
  1580. if not postIsAddressedToFollowers(baseDir,nickname,domain, \
  1581. port,httpPrefix,postJsonObject):
  1582. if debug:
  1583. print('Post is not addressed to followers')
  1584. return
  1585. print('Post is addressed to followers')
  1586. grouped=groupFollowersByDomain(baseDir,nickname,domain)
  1587. if not grouped:
  1588. if debug:
  1589. print('Post to followers did not resolve any domains')
  1590. return
  1591. print('Post to followers resolved domains')
  1592. print(str(grouped))
  1593. # this is after the message has arrived at the server
  1594. clientToServer=False
  1595. # for each instance
  1596. for followerDomain,followerHandles in grouped.items():
  1597. if debug:
  1598. print('DEBUG: follower handles for '+followerDomain)
  1599. pprint(followerHandles)
  1600. withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
  1601. if debug:
  1602. if withSharedInbox:
  1603. print(followerDomain+' has shared inbox')
  1604. else:
  1605. print(followerDomain+' does not have a shared inbox')
  1606. toPort=port
  1607. index=0
  1608. toDomain=followerHandles[index].split('@')[1]
  1609. if ':' in toDomain:
  1610. toPort=toDomain.split(':')[1]
  1611. toDomain=toDomain.split(':')[0]
  1612. cc=''
  1613. if withSharedInbox:
  1614. toNickname=followerHandles[index].split('@')[0]
  1615. # if there are more than one followers on the domain
  1616. # then send the post to the shared inbox
  1617. if len(followerHandles)>1:
  1618. toNickname='inbox'
  1619. if toNickname!='inbox' and postJsonObject.get('type'):
  1620. if postJsonObject['type']=='Update':
  1621. if postJsonObject.get('object'):
  1622. if isinstance(postJsonObject['object'], dict):
  1623. if postJsonObject['object'].get('type'):
  1624. if postJsonObject['object']['type']=='Person' or \
  1625. postJsonObject['object']['type']=='Application' or \
  1626. postJsonObject['object']['type']=='Group' or \
  1627. postJsonObject['object']['type']=='Service':
  1628. print('Sending profile update to shared inbox of '+toDomain)
  1629. toNickname='inbox'
  1630. if debug:
  1631. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1632. sendSignedJson(postJsonObject,session,baseDir, \
  1633. nickname,domain,port, \
  1634. toNickname,toDomain,toPort, \
  1635. cc,httpPrefix,True,clientToServer, \
  1636. federationList, \
  1637. sendThreads,postLog,cachedWebfingers, \
  1638. personCache,debug,projectVersion)
  1639. else:
  1640. # send to individual followers without using a shared inbox
  1641. for handle in followerHandles:
  1642. if debug:
  1643. print('DEBUG: Sending to '+handle)
  1644. toNickname=handle.split('@')[0]
  1645. if debug:
  1646. if postJsonObject['type']!='Update':
  1647. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1648. else:
  1649. print('DEBUG: Sending profile update from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1650. sendSignedJson(postJsonObject,session,baseDir, \
  1651. nickname,domain,port, \
  1652. toNickname,toDomain,toPort, \
  1653. cc,httpPrefix,True,clientToServer, \
  1654. federationList, \
  1655. sendThreads,postLog,cachedWebfingers, \
  1656. personCache,debug,projectVersion)
  1657. time.sleep(4)
  1658. if debug:
  1659. print('DEBUG: End of sendToFollowers')
  1660. def sendToFollowersThread(session,baseDir: str, \
  1661. nickname: str,domain: str,port: int, \
  1662. httpPrefix: str,federationList: [], \
  1663. sendThreads: [],postLog: [], \
  1664. cachedWebfingers: {},personCache: {}, \
  1665. postJsonObject: {},debug: bool, \
  1666. projectVersion: str):
  1667. """Returns a thread used to send a post to followers
  1668. """
  1669. sendThread= \
  1670. threadWithTrace(target=sendToFollowers, \
  1671. args=(session,baseDir, \
  1672. nickname,domain,port, \
  1673. httpPrefix,federationList, \
  1674. sendThreads,postLog, \
  1675. cachedWebfingers,personCache, \
  1676. postJsonObject.copy(),debug, \
  1677. projectVersion),daemon=True)
  1678. sendThread.start()
  1679. return sendThread
  1680. def createInbox(recentPostsCache: {}, \
  1681. session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1682. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1683. return createBoxIndexed(recentPostsCache, \
  1684. session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
  1685. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1686. def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1687. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1688. return createBoxIndexed({},session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
  1689. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1690. def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1691. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1692. return createBoxIndexed({},session,baseDir,'dm',nickname,domain,port,httpPrefix, \
  1693. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1694. def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1695. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1696. return createBoxIndexed({},session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
  1697. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1698. def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1699. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1700. return createBoxIndexed({},session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
  1701. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1702. def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1703. itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
  1704. return createBoxIndexed({},session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
  1705. itemsPerPage,headerOnly,authorized,False,pageNumber)
  1706. def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1707. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1708. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1709. boxname='moderation'
  1710. if port:
  1711. if port!=80 and port!=443:
  1712. if ':' not in domain:
  1713. domain=domain+':'+str(port)
  1714. if not pageNumber:
  1715. pageNumber=1
  1716. pageStr='?page='+str(pageNumber)
  1717. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1718. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1719. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1720. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1721. 'totalItems': 0,
  1722. 'type': 'OrderedCollection'}
  1723. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1724. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1725. 'orderedItems': [
  1726. ],
  1727. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1728. 'type': 'OrderedCollectionPage'}
  1729. if isModerator(baseDir,nickname):
  1730. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  1731. if os.path.isfile(moderationIndexFile):
  1732. with open(moderationIndexFile, "r") as f:
  1733. lines = f.readlines()
  1734. boxHeader['totalItems']=len(lines)
  1735. if headerOnly:
  1736. return boxHeader
  1737. pageLines=[]
  1738. if len(lines)>0:
  1739. endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
  1740. if endLineNumber<0:
  1741. endLineNumber=0
  1742. startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
  1743. if startLineNumber<0:
  1744. startLineNumber=0
  1745. lineNumber=startLineNumber
  1746. while lineNumber>=endLineNumber:
  1747. pageLines.append(lines[lineNumber].strip('\n'))
  1748. lineNumber-=1
  1749. for postUrl in pageLines:
  1750. postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
  1751. if os.path.isfile(postFilename):
  1752. postJsonObject=loadJson(postFilename)
  1753. if postJsonObject:
  1754. boxItems['orderedItems'].append(postJsonObject)
  1755. if headerOnly:
  1756. return boxHeader
  1757. return boxItems
  1758. def getStatusNumberFromPostFilename(filename) -> int:
  1759. """Gets the status number from a post filename
  1760. eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
  1761. returns 156295850695206
  1762. """
  1763. if '#statuses#' not in filename:
  1764. return None
  1765. return int(filename.split('#')[-1].replace('.json',''))
  1766. def isDM(postJsonObject: {}) -> bool:
  1767. """Returns true if the given post is a DM
  1768. """
  1769. if postJsonObject['type']!='Create':
  1770. return False
  1771. if not postJsonObject.get('object'):
  1772. return False
  1773. if not isinstance(postJsonObject['object'], dict):
  1774. return False
  1775. if postJsonObject['object']['type']!='Note':
  1776. return False
  1777. if postJsonObject['object'].get('moderationStatus'):
  1778. return False
  1779. fields=['to','cc']
  1780. for f in fields:
  1781. if not postJsonObject['object'].get(f):
  1782. continue
  1783. for toAddress in postJsonObject['object'][f]:
  1784. if toAddress.endswith('#Public'):
  1785. return False
  1786. if toAddress.endswith('followers'):
  1787. return False
  1788. return True
  1789. def isImageMedia(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {}) -> bool:
  1790. """Returns true if the given post has attached image media
  1791. """
  1792. if postJsonObject['type']=='Announce':
  1793. postJsonAnnounce=downloadAnnounce(session,baseDir,httpPrefix,nickname,domain,postJsonObject,__version__)
  1794. if postJsonAnnounce:
  1795. postJsonObject=postJsonAnnounce
  1796. if postJsonObject['type']!='Create':
  1797. return False
  1798. if not postJsonObject.get('object'):
  1799. return False
  1800. if not isinstance(postJsonObject['object'], dict):
  1801. return False
  1802. if postJsonObject['object'].get('moderationStatus'):
  1803. return False
  1804. if postJsonObject['object']['type']!='Note':
  1805. return False
  1806. if not postJsonObject['object'].get('attachment'):
  1807. return False
  1808. if not isinstance(postJsonObject['object']['attachment'], list):
  1809. return False
  1810. for attach in postJsonObject['object']['attachment']:
  1811. if attach.get('mediaType') and attach.get('url'):
  1812. if attach['mediaType'].startswith('image/') or \
  1813. attach['mediaType'].startswith('audio/') or \
  1814. attach['mediaType'].startswith('video/'):
  1815. return True
  1816. return False
  1817. def isReply(postJsonObject: {},actor: str) -> bool:
  1818. """Returns true if the given post is a reply to the given actor
  1819. """
  1820. if postJsonObject['type']!='Create':
  1821. return False
  1822. if not postJsonObject.get('object'):
  1823. return False
  1824. if not isinstance(postJsonObject['object'], dict):
  1825. return False
  1826. if postJsonObject['object'].get('moderationStatus'):
  1827. return False
  1828. if postJsonObject['object']['type']!='Note':
  1829. return False
  1830. if postJsonObject['object'].get('inReplyTo'):
  1831. if postJsonObject['object']['inReplyTo'].startswith(actor):
  1832. return True
  1833. if not postJsonObject['object'].get('tag'):
  1834. return False
  1835. if not isinstance(postJsonObject['object']['tag'], list):
  1836. return False
  1837. for tag in postJsonObject['object']['tag']:
  1838. if not tag.get('type'):
  1839. continue
  1840. if tag['type']=='Mention':
  1841. if not tag.get('href'):
  1842. continue
  1843. if actor in tag['href']:
  1844. return True
  1845. return False
  1846. def createBoxIndex(boxDir: str,postsInBoxDict: {}) -> int:
  1847. """ Creates an index for the given box
  1848. """
  1849. postsCtr=0
  1850. postsInPersonInbox=os.scandir(boxDir)
  1851. for postFilename in postsInPersonInbox:
  1852. postFilename=postFilename.name
  1853. if not postFilename.endswith('.json'):
  1854. continue
  1855. # extract the status number
  1856. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1857. if statusNumber:
  1858. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1859. postsCtr+=1
  1860. return postsCtr
  1861. def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
  1862. postsInBoxDict: {},postsCtr: int, \
  1863. nickname: str,domain: str, \
  1864. ocapAlways: bool) -> int:
  1865. """ Creates an index for the given shared inbox
  1866. """
  1867. handle=nickname+'@'+domain
  1868. followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
  1869. postsInSharedInbox=os.scandir(sharedBoxDir)
  1870. followingHandles=None
  1871. for postFilename in postsInSharedInbox:
  1872. postFilename=postFilename.name
  1873. if not postFilename.endswith('.json'):
  1874. continue
  1875. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1876. if not statusNumber:
  1877. continue
  1878. sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
  1879. # get the actor from the shared post
  1880. postJsonObject=loadJson(sharedInboxFilename,0)
  1881. if not postJsonObject:
  1882. print('WARN: json load exception createSharedInboxIndex')
  1883. continue
  1884. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  1885. if not actorNickname:
  1886. continue
  1887. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  1888. if not actorDomain:
  1889. continue
  1890. # is the actor followed by this account?
  1891. if not followingHandles:
  1892. with open(followingFilename, 'r') as followingFile:
  1893. followingHandles = followingFile.read()
  1894. if actorNickname+'@'+actorDomain not in followingHandles:
  1895. continue
  1896. if ocapAlways:
  1897. capsList=None
  1898. # Note: should this be in the Create or the object of a post?
  1899. if postJsonObject.get('capability'):
  1900. if isinstance(postJsonObject['capability'], list):
  1901. capsList=postJsonObject['capability']
  1902. # Have capabilities been granted for the sender?
  1903. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
  1904. if not os.path.isfile(ocapFilename):
  1905. continue
  1906. # read the capabilities id
  1907. ocapJson=loadJson(ocapFilename,0)
  1908. if not ocapJson:
  1909. print('WARN: json load exception createSharedInboxIndex')
  1910. else:
  1911. if ocapJson.get('id'):
  1912. if ocapJson['id'] in capsList:
  1913. postsInBoxDict[statusNumber]=sharedInboxFilename
  1914. postsCtr+=1
  1915. else:
  1916. postsInBoxDict[statusNumber]=sharedInboxFilename
  1917. postsCtr+=1
  1918. return postsCtr
  1919. def addPostStringToTimeline(postStr: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
  1920. """ is this a valid timeline post?
  1921. """
  1922. # must be a "Note" or "Announce" type
  1923. if '"Note"' in postStr or '"Announce"' in postStr or \
  1924. ('"Question"' in postStr and ('"Create"' in postStr or '"Update"' in postStr)):
  1925. if boxname=='dm':
  1926. if '#Public' in postStr or '/followers' in postStr:
  1927. return False
  1928. elif boxname=='tlreplies':
  1929. if boxActor not in postStr:
  1930. return False
  1931. elif boxname=='tlmedia':
  1932. if '"Create"' in postStr:
  1933. if 'mediaType' not in postStr or 'image/' not in postStr:
  1934. return False
  1935. # add the post to the dictionary
  1936. postsInBox.append(postStr)
  1937. return True
  1938. return False
  1939. def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
  1940. """ Reads a post from file and decides whether it is valid
  1941. """
  1942. with open(filePath, 'r') as postFile:
  1943. postStr = postFile.read()
  1944. return addPostStringToTimeline(postStr,boxname,postsInBox,boxActor)
  1945. return False
  1946. def createBoxIndexed(recentPostsCache: {}, \
  1947. session,baseDir: str,boxname: str, \
  1948. nickname: str,domain: str,port: int,httpPrefix: str, \
  1949. itemsPerPage: int,headerOnly: bool,authorized :bool, \
  1950. ocapAlways: bool,pageNumber=None) -> {}:
  1951. """Constructs the box feed for a person with the given nickname
  1952. """
  1953. if not authorized or not pageNumber:
  1954. pageNumber=1
  1955. if boxname!='inbox' and boxname!='dm' and \
  1956. boxname!='tlreplies' and boxname!='tlmedia' and \
  1957. boxname!='outbox' and boxname!='tlbookmarks':
  1958. return None
  1959. if boxname!='dm' and boxname!='tlreplies' and \
  1960. boxname!='tlmedia' and boxname!='tlbookmarks':
  1961. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1962. else:
  1963. # extract DMs or replies or media from the inbox
  1964. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1965. announceCacheDir=baseDir+'/cache/announce/'+nickname
  1966. sharedBoxDir=None
  1967. if boxname=='inbox' or boxname=='tlreplies' or \
  1968. boxname=='tlmedia':
  1969. sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
  1970. # bookmarks timeline is like the inbox but has its own separate index
  1971. indexBoxName=boxname
  1972. if boxname=='tlbookmarks':
  1973. indexBoxName='bookmarks'
  1974. elif boxname=='dm':
  1975. indexBoxName='dm'
  1976. elif boxname=='tlreplies':
  1977. indexBoxName='tlreplies'
  1978. elif boxname=='tlmedia':
  1979. indexBoxName='tlmedia'
  1980. if port:
  1981. if port!=80 and port!=443:
  1982. if ':' not in domain:
  1983. domain=domain+':'+str(port)
  1984. boxActor=httpPrefix+'://'+domain+'/users/'+nickname
  1985. pageStr='?page=true'
  1986. if pageNumber:
  1987. try:
  1988. pageStr='?page='+str(pageNumber)
  1989. except:
  1990. pass
  1991. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1992. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1993. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1994. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1995. 'totalItems': 0,
  1996. 'type': 'OrderedCollection'}
  1997. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1998. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1999. 'orderedItems': [
  2000. ],
  2001. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  2002. 'type': 'OrderedCollectionPage'}
  2003. postsInBox=[]
  2004. indexFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+indexBoxName+'.index'
  2005. postsCtr=0
  2006. if os.path.isfile(indexFilename):
  2007. maxPostCtr=itemsPerPage*pageNumber
  2008. with open(indexFilename, 'r') as indexFile:
  2009. while postsCtr<maxPostCtr:
  2010. postFilename=indexFile.readline()
  2011. if not postFilename:
  2012. postsCtr+=1
  2013. continue
  2014. # Skip through any posts previous to the current page
  2015. if postsCtr<int((pageNumber-1)*itemsPerPage):
  2016. postsCtr+=1
  2017. continue
  2018. # if this is a full path then remove the directories
  2019. if '/' in postFilename:
  2020. postFilename=postFilename.split('/')[-1]
  2021. # filename of the post without any extension or path
  2022. # This should also correspond to any index entry in the posts cache
  2023. postUrl=postFilename.replace('\n','').replace('.json','').strip()
  2024. postAdded=False
  2025. # is the post cached in memory?
  2026. if recentPostsCache.get('index'):
  2027. if postUrl in recentPostsCache['index']:
  2028. if recentPostsCache['json'].get(postUrl):
  2029. addPostStringToTimeline(recentPostsCache['json'][postUrl], \
  2030. boxname,postsInBox,boxActor)
  2031. postAdded=True
  2032. if not postAdded:
  2033. # read the post from file
  2034. fullPostFilename= \
  2035. locatePost(baseDir,nickname,domain,postUrl,False)
  2036. if fullPostFilename:
  2037. addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
  2038. else:
  2039. print('WARN: unable to locate post '+postUrl)
  2040. postsCtr+=1
  2041. # Generate first and last entries within header
  2042. if postsCtr>0:
  2043. lastPage=int(postsCtr/itemsPerPage)
  2044. if lastPage<1:
  2045. lastPage=1
  2046. boxHeader['last']= \
  2047. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
  2048. if headerOnly:
  2049. boxHeader['totalItems']=len(postsInBox)
  2050. prevPageStr='true'
  2051. if pageNumber>1:
  2052. prevPageStr=str(pageNumber-1)
  2053. boxHeader['prev']= \
  2054. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+prevPageStr
  2055. nextPageStr=str(pageNumber+1)
  2056. boxHeader['next']= \
  2057. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+nextPageStr
  2058. return boxHeader
  2059. for postStr in postsInBox:
  2060. p=None
  2061. try:
  2062. p=json.loads(postStr)
  2063. except:
  2064. continue
  2065. # remove any capability so that it's not displayed
  2066. if p.get('capability'):
  2067. del p['capability']
  2068. # Don't show likes, replies or shares (announces) to unauthorized viewers
  2069. if not authorized:
  2070. if p.get('object'):
  2071. if isinstance(p['object'], dict):
  2072. if p['object'].get('likes'):
  2073. p['likes']={'items': []}
  2074. if p['object'].get('replies'):
  2075. p['replies']={}
  2076. if p['object'].get('shares'):
  2077. p['shares']={}
  2078. if p['object'].get('bookmarks'):
  2079. p['bookmarks']={}
  2080. boxItems['orderedItems'].append(p)
  2081. return boxItems
  2082. def expireCache(baseDir: str,personCache: {},httpPrefix: str,archiveDir: str,maxPostsInBox=32000):
  2083. """Thread used to expire actors from the cache and archive old posts
  2084. """
  2085. while True:
  2086. # once per day
  2087. time.sleep(60*60*24)
  2088. expirePersonCache(basedir,personCache)
  2089. archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
  2090. def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str,maxPostsInBox=32000) -> None:
  2091. """Archives posts for all accounts
  2092. """
  2093. if archiveDir:
  2094. if not os.path.isdir(archiveDir):
  2095. os.mkdir(archiveDir)
  2096. if archiveDir:
  2097. if not os.path.isdir(archiveDir+'/accounts'):
  2098. os.mkdir(archiveDir+'/accounts')
  2099. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  2100. for handle in dirs:
  2101. if '@' in handle:
  2102. nickname=handle.split('@')[0]
  2103. domain=handle.split('@')[1]
  2104. archiveSubdir=None
  2105. if archiveDir:
  2106. if not os.path.isdir(archiveDir+'/accounts/'+handle):
  2107. os.mkdir(archiveDir+'/accounts/'+handle)
  2108. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
  2109. os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
  2110. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
  2111. os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
  2112. archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
  2113. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2114. 'inbox',archiveSubdir, \
  2115. maxPostsInBox)
  2116. if archiveDir:
  2117. archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
  2118. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2119. 'outbox',archiveSubdir, \
  2120. maxPostsInBox)
  2121. def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
  2122. boxname: str,archiveDir: str,maxPostsInBox=32000) -> None:
  2123. """Retain a maximum number of posts within the given box
  2124. Move any others to an archive directory
  2125. """
  2126. if boxname!='inbox' and boxname!='outbox':
  2127. return
  2128. if archiveDir:
  2129. if not os.path.isdir(archiveDir):
  2130. os.mkdir(archiveDir)
  2131. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  2132. postsInBox=os.scandir(boxDir)
  2133. noOfPosts=0
  2134. for f in postsInBox:
  2135. noOfPosts+=1
  2136. if noOfPosts<=maxPostsInBox:
  2137. return
  2138. # remove entries from the index
  2139. handle=nickname+'@'+domain
  2140. indexFilename=baseDir+'/accounts/'+handle+'/'+boxname+'.index'
  2141. if os.path.isfile(indexFilename):
  2142. indexCtr=0
  2143. # get the existing index entries as a string
  2144. newIndex=''
  2145. with open(indexFilename, 'r') as indexFile:
  2146. for postId in indexFile:
  2147. newIndex+=postId
  2148. indexCtr+=1
  2149. if indexCtr>=maxPostsInBox:
  2150. break
  2151. # save the new index file
  2152. if len(newIndex)>0:
  2153. indexFile=open(indexFilename,'w+')
  2154. if indexFile:
  2155. indexFile.write(newIndex)
  2156. indexFile.close()
  2157. postsInBoxDict={}
  2158. postsCtr=0
  2159. postsInBox=os.scandir(boxDir)
  2160. for postFilename in postsInBox:
  2161. postFilename=postFilename.name
  2162. if not postFilename.endswith('.json'):
  2163. continue
  2164. # Time of file creation
  2165. fullFilename=os.path.join(boxDir,postFilename)
  2166. if os.path.isfile(fullFilename):
  2167. content=open(fullFilename).read()
  2168. if '"published":' in content:
  2169. publishedStr=content.split('"published":')[1]
  2170. if '"' in publishedStr:
  2171. publishedStr=publishedStr.split('"')[1]
  2172. if publishedStr.endswith('Z'):
  2173. postsInBoxDict[publishedStr]=postFilename
  2174. postsCtr+=1
  2175. noOfPosts=postsCtr
  2176. if noOfPosts<=maxPostsInBox:
  2177. return
  2178. # sort the list in ascending order of date
  2179. postsInBoxSorted=OrderedDict(sorted(postsInBoxDict.items(),reverse=False))
  2180. # directory containing cached html posts
  2181. postCacheDir=boxDir.replace('/'+boxname,'/postcache')
  2182. for publishedStr,postFilename in postsInBoxSorted.items():
  2183. filePath=os.path.join(boxDir,postFilename)
  2184. if not os.path.isfile(filePath):
  2185. continue
  2186. if archiveDir:
  2187. repliesPath=filePath.replace('.json','.replies')
  2188. archivePath=os.path.join(archiveDir,postFilename)
  2189. os.rename(filePath,archivePath)
  2190. if os.path.isfile(repliesPath):
  2191. os.rename(repliesPath,archivePath)
  2192. else:
  2193. deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
  2194. # remove cached html posts
  2195. postCacheFilename=os.path.join(postCacheDir,postFilename).replace('.json','.html')
  2196. if os.path.isfile(postCacheFilename):
  2197. os.remove(postCacheFilename)
  2198. noOfPosts-=1
  2199. if noOfPosts<=maxPostsInBox:
  2200. break
  2201. def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
  2202. raw: bool,simple: bool,useTor: bool, \
  2203. port: int,httpPrefix: str, \
  2204. debug: bool,projectVersion: str) -> None:
  2205. """ This is really just for test purposes
  2206. """
  2207. session = createSession(useTor)
  2208. personCache={}
  2209. cachedWebfingers={}
  2210. federationList=[]
  2211. domainFull=domain
  2212. if port:
  2213. if port!=80 and port!=443:
  2214. if ':' not in domain:
  2215. domainFull=domain+':'+str(port)
  2216. handle=httpPrefix+"://"+domainFull+"/@"+nickname
  2217. wfRequest = \
  2218. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  2219. domain,projectVersion)
  2220. if not wfRequest:
  2221. sys.exit()
  2222. personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
  2223. getPersonBox(baseDir,session,wfRequest,personCache, \
  2224. projectVersion,httpPrefix,nickname,domain,'outbox')
  2225. wfResult = json.dumps(wfRequest, indent=2, sort_keys=False)
  2226. maxMentions=10
  2227. maxEmoji=10
  2228. maxAttachments=5
  2229. userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
  2230. maxAttachments,federationList, \
  2231. personCache,raw,simple,debug, \
  2232. projectVersion,httpPrefix,domain)
  2233. #print(str(userPosts))
  2234. def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
  2235. nickname: str,domain: str,port: int, \
  2236. followerUrl,updateCaps: [], \
  2237. sendThreads: [],postLog: [], \
  2238. cachedWebfingers: {},personCache: {}, \
  2239. federationList :[],debug :bool, \
  2240. projectVersion: str) -> int:
  2241. """When the capabilities for a follower are changed this
  2242. sends out an update. followerUrl is the actor of the follower.
  2243. """
  2244. updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
  2245. nickname,domain,port, \
  2246. followerUrl, \
  2247. updateCaps)
  2248. if not updateJson:
  2249. return 1
  2250. if debug:
  2251. pprint(updateJson)
  2252. print('DEBUG: sending capabilities update from '+ \
  2253. nickname+'@'+domain+' port '+ str(port) + \
  2254. ' to '+followerUrl)
  2255. clientToServer=False
  2256. followerNickname=getNicknameFromActor(followerUrl)
  2257. if not followerNickname:
  2258. print('WARN: unable to find nickname in '+followerUrl)
  2259. return 1
  2260. followerDomain,followerPort=getDomainFromActor(followerUrl)
  2261. return sendSignedJson(updateJson,session,baseDir, \
  2262. nickname,domain,port, \
  2263. followerNickname,followerDomain,followerPort, '', \
  2264. httpPrefix,True,clientToServer, \
  2265. federationList, \
  2266. sendThreads,postLog,cachedWebfingers, \
  2267. personCache,debug,projectVersion)
  2268. def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
  2269. postRepliesFilename: str,authorized: bool, \
  2270. repliesJson: {}) -> None:
  2271. # populate the items list with replies
  2272. repliesBoxes=['outbox','inbox']
  2273. with open(postRepliesFilename,'r') as repliesFile:
  2274. for messageId in repliesFile:
  2275. replyFound=False
  2276. # examine inbox and outbox
  2277. for boxname in repliesBoxes:
  2278. searchFilename= \
  2279. baseDir+ \
  2280. '/accounts/'+nickname+'@'+ \
  2281. domain+'/'+ \
  2282. boxname+'/'+ \
  2283. messageId.replace('\n','').replace('/','#')+'.json'
  2284. if os.path.isfile(searchFilename):
  2285. if authorized or \
  2286. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2287. postJsonObject=loadJson(searchFilename)
  2288. if postJsonObject:
  2289. if postJsonObject['object'].get('cc'):
  2290. if authorized or \
  2291. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2292. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2293. repliesJson['orderedItems'].append(postJsonObject)
  2294. replyFound=True
  2295. else:
  2296. if authorized or \
  2297. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2298. repliesJson['orderedItems'].append(postJsonObject)
  2299. replyFound=True
  2300. break
  2301. # if not in either inbox or outbox then examine the shared inbox
  2302. if not replyFound:
  2303. searchFilename= \
  2304. baseDir+ \
  2305. '/accounts/inbox@'+ \
  2306. domain+'/inbox/'+ \
  2307. messageId.replace('\n','').replace('/','#')+'.json'
  2308. if os.path.isfile(searchFilename):
  2309. if authorized or \
  2310. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2311. # get the json of the reply and append it to the collection
  2312. postJsonObject=loadJson(searchFilename)
  2313. if postJsonObject:
  2314. if postJsonObject['object'].get('cc'):
  2315. if authorized or \
  2316. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2317. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2318. repliesJson['orderedItems'].append(postJsonObject)
  2319. else:
  2320. if authorized or \
  2321. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2322. repliesJson['orderedItems'].append(postJsonObject)
  2323. def rejectAnnounce(announceFilename: str):
  2324. """Marks an announce as rejected
  2325. """
  2326. if not os.path.isfile(announceFilename+'.reject'):
  2327. rejectAnnounceFile=open(announceFilename+'.reject', "w+")
  2328. rejectAnnounceFile.write('\n')
  2329. rejectAnnounceFile.close()
  2330. def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {},projectVersion: str) -> {}:
  2331. """Download the post referenced by an announce
  2332. """
  2333. if not postJsonObject.get('object'):
  2334. return None
  2335. if not isinstance(postJsonObject['object'], str):
  2336. return None
  2337. # get the announced post
  2338. announceCacheDir=baseDir+'/cache/announce/'+nickname
  2339. if not os.path.isdir(announceCacheDir):
  2340. os.mkdir(announceCacheDir)
  2341. announceFilename=announceCacheDir+'/'+postJsonObject['object'].replace('/','#')+'.json'
  2342. print('announceFilename: '+announceFilename)
  2343. if os.path.isfile(announceFilename+'.reject'):
  2344. return None
  2345. if os.path.isfile(announceFilename):
  2346. print('Reading cached Announce content for '+postJsonObject['object'])
  2347. postJsonObject=loadJson(announceFilename)
  2348. if postJsonObject:
  2349. return postJsonObject
  2350. else:
  2351. print('Downloading Announce content for '+postJsonObject['object'])
  2352. asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  2353. if '/channel/' in postJsonObject['actor']:
  2354. asHeader={'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  2355. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  2356. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  2357. announcedJson=getJson(session,postJsonObject['object'],asHeader,None,projectVersion,httpPrefix,domain)
  2358. if not announcedJson:
  2359. return None
  2360. if not isinstance(announcedJson, dict):
  2361. print('WARN: announce json is not a dict - '+postJsonObject['object'])
  2362. rejectAnnounce(announceFilename)
  2363. return None
  2364. if not announcedJson.get('id'):
  2365. rejectAnnounce(announceFilename)
  2366. return None
  2367. if '/statuses/' not in announcedJson['id']:
  2368. rejectAnnounce(announceFilename)
  2369. return None
  2370. if '/users/' not in announcedJson['id'] and \
  2371. '/channel/' not in announcedJson['id'] and \
  2372. '/profile/' not in announcedJson['id']:
  2373. rejectAnnounce(announceFilename)
  2374. return None
  2375. if not announcedJson.get('type'):
  2376. rejectAnnounce(announceFilename)
  2377. #pprint(announcedJson)
  2378. return None
  2379. if announcedJson['type']!='Note':
  2380. rejectAnnounce(announceFilename)
  2381. #pprint(announcedJson)
  2382. return None
  2383. # wrap in create to be consistent with other posts
  2384. announcedJson= \
  2385. outboxMessageCreateWrap(httpPrefix, \
  2386. actorNickname,actorDomain,actorPort, \
  2387. announcedJson)
  2388. if announcedJson['type']!='Create':
  2389. rejectAnnounce(announceFilename)
  2390. #pprint(announcedJson)
  2391. return None
  2392. # set the id to the original status
  2393. announcedJson['id']=postJsonObject['object']
  2394. announcedJson['object']['id']=postJsonObject['object']
  2395. # check that the repeat isn't for a blocked account
  2396. attributedNickname=getNicknameFromActor(announcedJson['object']['id'])
  2397. attributedDomain,attributedPort=getDomainFromActor(announcedJson['object']['id'])
  2398. if attributedNickname and attributedDomain:
  2399. if attributedPort:
  2400. if attributedPort!=80 and attributedPort!=443:
  2401. attributedDomain=attributedDomain+':'+str(attributedPort)
  2402. if isBlocked(baseDir,nickname,domain,attributedNickname,attributedDomain):
  2403. rejectAnnounce(announceFilename)
  2404. return None
  2405. postJsonObject=announcedJson
  2406. if saveJson(postJsonObject,announceFilename):
  2407. return postJsonObject
  2408. return None
  2409. def mutePost(baseDir: str,nickname: str,domain: str,postId: str, \
  2410. recentPostsCache: {}) -> None:
  2411. """ Mutes the given post
  2412. """
  2413. postFilename=locatePost(baseDir,nickname,domain,postId)
  2414. if not postFilename:
  2415. return
  2416. postJsonObject=loadJson(postFilename)
  2417. if not postJsonObject:
  2418. return
  2419. print('MUTE: '+postFilename)
  2420. muteFile=open(postFilename+'.muted', "w")
  2421. if muteFile:
  2422. muteFile.write('\n')
  2423. muteFile.close()
  2424. # remove cached posts so that the muted version gets created
  2425. cachedPostFilename= \
  2426. getCachedPostFilename(baseDir,nickname,domain,postJsonObject)
  2427. if cachedPostFilename:
  2428. if os.path.isfile(cachedPostFilename):
  2429. os.remove(cachedPostFilename)
  2430. # if the post is in the recent posts cache then mark it as muted
  2431. if recentPostsCache.get('index'):
  2432. postId=postJsonObject['id'].replace('/activity','').replace('/','#')
  2433. if postId in recentPostsCache['index']:
  2434. print('MUTE: '+postId+' is in recent posts cache')
  2435. if recentPostsCache['json'].get(postId):
  2436. postJsonObject['muted']=True
  2437. recentPostsCache['json'][postId]=json.dumps(postJsonObject)
  2438. print('MUTE: '+postId+' marked as muted in recent posts cache')
  2439. def unmutePost(baseDir: str,nickname: str,domain: str,postId: str, \
  2440. recentPostsCache: {}) -> None:
  2441. """ Unmutes the given post
  2442. """
  2443. postFilename=locatePost(baseDir,nickname,domain,postId)
  2444. if not postFilename:
  2445. return
  2446. postJsonObject=loadJson(postFilename)
  2447. if not postJsonObject:
  2448. return
  2449. print('UNMUTE: '+postFilename)
  2450. muteFilename=postFilename+'.muted'
  2451. if os.path.isfile(muteFilename):
  2452. os.remove(muteFilename)
  2453. # remove cached posts so that it gets recreated
  2454. cachedPostFilename= \
  2455. getCachedPostFilename(baseDir,nickname,domain,postJsonObject)
  2456. if cachedPostFilename:
  2457. if os.path.isfile(cachedPostFilename):
  2458. os.remove(cachedPostFilename)
  2459. removePostFromCache(postJsonObject,recentPostsCache)