posts.py 102 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import requests
  9. import json
  10. import commentjson
  11. import html
  12. import datetime
  13. import os
  14. import shutil
  15. import threading
  16. import sys
  17. import trace
  18. import time
  19. from time import gmtime, strftime
  20. from collections import OrderedDict
  21. from threads import threadWithTrace
  22. from cache import storePersonInCache
  23. from cache import getPersonFromCache
  24. from cache import expirePersonCache
  25. from pprint import pprint
  26. from random import randint
  27. from session import createSession
  28. from session import getJson
  29. from session import postJsonString
  30. from session import postImage
  31. from webfinger import webfingerHandle
  32. from httpsig import createSignedHeader
  33. from utils import getStatusNumber
  34. from utils import createPersonDir
  35. from utils import urlPermitted
  36. from utils import getNicknameFromActor
  37. from utils import getDomainFromActor
  38. from utils import deletePost
  39. from utils import validNickname
  40. from utils import locatePost
  41. from utils import loadJson
  42. from utils import saveJson
  43. from capabilities import getOcapFilename
  44. from capabilities import capabilitiesUpdate
  45. from media import attachMedia
  46. from content import addHtmlTags
  47. from content import replaceEmojiFromTags
  48. from auth import createBasicAuthHeader
  49. from config import getConfigParam
  50. from blocking import isBlocked
  51. try:
  52. from BeautifulSoup import BeautifulSoup
  53. except ImportError:
  54. from bs4 import BeautifulSoup
  55. def isModerator(baseDir: str,nickname: str) -> bool:
  56. """Returns true if the given nickname is a moderator
  57. """
  58. moderatorsFile=baseDir+'/accounts/moderators.txt'
  59. if not os.path.isfile(moderatorsFile):
  60. if getConfigParam(baseDir,'admin')==nickname:
  61. return True
  62. return False
  63. with open(moderatorsFile, "r") as f:
  64. lines = f.readlines()
  65. if len(lines)==0:
  66. if getConfigParam(baseDir,'admin')==nickname:
  67. return True
  68. for moderator in lines:
  69. moderator=moderator.strip('\n')
  70. if moderator==nickname:
  71. return True
  72. return False
  73. def noOfFollowersOnDomain(baseDir: str,handle: str, \
  74. domain: str, followFile='followers.txt') -> int:
  75. """Returns the number of followers of the given handle from the given domain
  76. """
  77. filename=baseDir+'/accounts/'+handle+'/'+followFile
  78. if not os.path.isfile(filename):
  79. return 0
  80. ctr=0
  81. with open(filename, "r") as followersFilename:
  82. for followerHandle in followersFilename:
  83. if '@' in followerHandle:
  84. followerDomain= \
  85. followerHandle.split('@')[1].replace('\n','')
  86. if domain==followerDomain:
  87. ctr+=1
  88. return ctr
  89. def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
  90. debug=False):
  91. """Returns the public or private key of a person
  92. """
  93. handle=nickname+'@'+domain
  94. keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
  95. if not os.path.isfile(keyFilename):
  96. if debug:
  97. print('DEBUG: private key file not found: '+keyFilename)
  98. return ''
  99. keyPem=''
  100. with open(keyFilename, "r") as pemFile:
  101. keyPem=pemFile.read()
  102. if len(keyPem)<20:
  103. if debug:
  104. print('DEBUG: private key was too short: '+keyPem)
  105. return ''
  106. return keyPem
  107. def cleanHtml(rawHtml: str) -> str:
  108. text = BeautifulSoup(rawHtml, 'html.parser').get_text()
  109. return html.unescape(text)
  110. def getUserUrl(wfRequest: {}) -> str:
  111. if wfRequest.get('links'):
  112. for link in wfRequest['links']:
  113. if link.get('type') and link.get('href'):
  114. if link['type'] == 'application/activity+json':
  115. if not ('/users/' in link['href'] or \
  116. '/profile/' in link['href'] or \
  117. '/channel/' in link['href']):
  118. print('Webfinger activity+json contains single user instance actor')
  119. return link['href']
  120. return None
  121. def parseUserFeed(session,feedUrl: str,asHeader: {}, \
  122. projectVersion: str,httpPrefix: str,domain: str) -> None:
  123. feedJson = getJson(session,feedUrl,asHeader,None, \
  124. projectVersion,httpPrefix,domain)
  125. if not feedJson:
  126. return
  127. if 'orderedItems' in feedJson:
  128. for item in feedJson['orderedItems']:
  129. yield item
  130. nextUrl = None
  131. if 'first' in feedJson:
  132. nextUrl = feedJson['first']
  133. elif 'next' in feedJson:
  134. nextUrl = feedJson['next']
  135. if nextUrl:
  136. if isinstance(nextUrl, str):
  137. userFeed=parseUserFeed(session,nextUrl,asHeader, \
  138. projectVersion,httpPrefix,domain)
  139. for item in userFeed:
  140. yield item
  141. elif isinstance(nextUrl, dict):
  142. userFeed=nextUrl
  143. if userFeed.get('orderedItems'):
  144. for item in userFeed['orderedItems']:
  145. yield item
  146. def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
  147. projectVersion: str,httpPrefix: str, \
  148. nickname: str,domain: str, \
  149. boxName='inbox') -> (str,str,str,str,str,str,str,str):
  150. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  151. if not wfRequest.get('errors'):
  152. personUrl = getUserUrl(wfRequest)
  153. else:
  154. if nickname=='dev':
  155. # try single user instance
  156. print('getPersonBox: Trying single user instance with ld+json')
  157. personUrl = httpPrefix+'://'+domain
  158. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  159. else:
  160. personUrl = httpPrefix+'://'+domain+'/users/'+nickname
  161. if not personUrl:
  162. return None,None,None,None,None,None,None,None
  163. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  164. if not personJson:
  165. if '/channel/' in personUrl:
  166. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  167. personJson = getJson(session,personUrl,asHeader,None, \
  168. projectVersion,httpPrefix,domain)
  169. if not personJson:
  170. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  171. personJson = getJson(session,personUrl,asHeader,None, \
  172. projectVersion,httpPrefix,domain)
  173. if not personJson:
  174. print('Unable to get actor')
  175. return None,None,None,None,None,None,None,None
  176. boxJson=None
  177. if not personJson.get(boxName):
  178. if personJson.get('endpoints'):
  179. if personJson['endpoints'].get(boxName):
  180. boxJson=personJson['endpoints'][boxName]
  181. else:
  182. boxJson=personJson[boxName]
  183. if not boxJson:
  184. return None,None,None,None,None,None,None,None
  185. personId=None
  186. if personJson.get('id'):
  187. personId=personJson['id']
  188. pubKeyId=None
  189. pubKey=None
  190. if personJson.get('publicKey'):
  191. if personJson['publicKey'].get('id'):
  192. pubKeyId=personJson['publicKey']['id']
  193. if personJson['publicKey'].get('publicKeyPem'):
  194. pubKey=personJson['publicKey']['publicKeyPem']
  195. sharedInbox=None
  196. if personJson.get('sharedInbox'):
  197. sharedInbox=personJson['sharedInbox']
  198. else:
  199. if personJson.get('endpoints'):
  200. if personJson['endpoints'].get('sharedInbox'):
  201. sharedInbox=personJson['endpoints']['sharedInbox']
  202. capabilityAcquisition=None
  203. if personJson.get('capabilityAcquisitionEndpoint'):
  204. capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
  205. avatarUrl=None
  206. if personJson.get('icon'):
  207. if personJson['icon'].get('url'):
  208. avatarUrl=personJson['icon']['url']
  209. displayName=None
  210. if personJson.get('name'):
  211. displayName=personJson['name']
  212. storePersonInCache(baseDir,personUrl,personJson,personCache)
  213. return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
  214. def getPosts(session,outboxUrl: str,maxPosts: int, \
  215. maxMentions: int, \
  216. maxEmoji: int,maxAttachments: int, \
  217. federationList: [], \
  218. personCache: {},raw: bool, \
  219. simple: bool,debug: bool, \
  220. projectVersion: str,httpPrefix: str,domain: str) -> {}:
  221. """Gets public posts from an outbox
  222. """
  223. personPosts={}
  224. if not outboxUrl:
  225. return personPosts
  226. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  227. if '/outbox/' in outboxUrl:
  228. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  229. if raw:
  230. result = []
  231. i = 0
  232. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  233. projectVersion,httpPrefix,domain)
  234. for item in userFeed:
  235. result.append(item)
  236. i += 1
  237. if i == maxPosts:
  238. break
  239. pprint(result)
  240. return None
  241. i = 0
  242. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  243. projectVersion,httpPrefix,domain)
  244. for item in userFeed:
  245. if not item.get('id'):
  246. if debug:
  247. print('No id')
  248. continue
  249. if not item.get('type'):
  250. if debug:
  251. print('No type')
  252. continue
  253. if item['type'] != 'Create':
  254. if debug:
  255. print('Not Create type')
  256. continue
  257. if not item.get('object'):
  258. if debug:
  259. print('No object')
  260. continue
  261. if not isinstance(item['object'], dict):
  262. if debug:
  263. print('item object is not a dict')
  264. continue
  265. if not item['object'].get('published'):
  266. if debug:
  267. print('No published attribute')
  268. continue
  269. #pprint(item)
  270. published = item['object']['published']
  271. if not personPosts.get(item['id']):
  272. # check that this is a public post
  273. # #Public should appear in the "to" list
  274. if item['object'].get('to'):
  275. isPublic=False
  276. for recipient in item['object']['to']:
  277. if recipient.endswith('#Public'):
  278. isPublic=True
  279. break
  280. if not isPublic:
  281. continue
  282. content = item['object']['content'].replace('&apos;',"'")
  283. mentions=[]
  284. emoji={}
  285. if item['object'].get('tag'):
  286. for tagItem in item['object']['tag']:
  287. tagType=tagItem['type'].lower()
  288. if tagType=='emoji':
  289. if tagItem.get('name') and tagItem.get('icon'):
  290. if tagItem['icon'].get('url'):
  291. # No emoji from non-permitted domains
  292. if urlPermitted(tagItem['icon']['url'], \
  293. federationList, \
  294. "objects:read"):
  295. emojiName=tagItem['name']
  296. emojiIcon=tagItem['icon']['url']
  297. emoji[emojiName]=emojiIcon
  298. else:
  299. if debug:
  300. print('url not permitted '+tagItem['icon']['url'])
  301. if tagType=='mention':
  302. if tagItem.get('name'):
  303. if tagItem['name'] not in mentions:
  304. mentions.append(tagItem['name'])
  305. if len(mentions)>maxMentions:
  306. if debug:
  307. print('max mentions reached')
  308. continue
  309. if len(emoji)>maxEmoji:
  310. if debug:
  311. print('max emojis reached')
  312. continue
  313. summary = ''
  314. if item['object'].get('summary'):
  315. if item['object']['summary']:
  316. summary = item['object']['summary']
  317. inReplyTo = ''
  318. if item['object'].get('inReplyTo'):
  319. if item['object']['inReplyTo']:
  320. # No replies to non-permitted domains
  321. if not urlPermitted(item['object']['inReplyTo'], \
  322. federationList, \
  323. "objects:read"):
  324. if debug:
  325. print('url not permitted '+item['object']['inReplyTo'])
  326. continue
  327. inReplyTo = item['object']['inReplyTo']
  328. conversation = ''
  329. if item['object'].get('conversation'):
  330. if item['object']['conversation']:
  331. # no conversations originated in non-permitted domains
  332. if urlPermitted(item['object']['conversation'], \
  333. federationList,"objects:read"):
  334. conversation = item['object']['conversation']
  335. attachment = []
  336. if item['object'].get('attachment'):
  337. if item['object']['attachment']:
  338. for attach in item['object']['attachment']:
  339. if attach.get('name') and attach.get('url'):
  340. # no attachments from non-permitted domains
  341. if urlPermitted(attach['url'], \
  342. federationList, \
  343. "objects:read"):
  344. attachment.append([attach['name'],attach['url']])
  345. else:
  346. if debug:
  347. print('url not permitted '+attach['url'])
  348. sensitive = False
  349. if item['object'].get('sensitive'):
  350. sensitive = item['object']['sensitive']
  351. if simple:
  352. print(cleanHtml(content)+'\n')
  353. else:
  354. pprint(item)
  355. personPosts[item['id']] = {
  356. "sensitive": sensitive,
  357. "inreplyto": inReplyTo,
  358. "summary": summary,
  359. "html": content,
  360. "plaintext": cleanHtml(content),
  361. "attachment": attachment,
  362. "mentions": mentions,
  363. "emoji": emoji,
  364. "conversation": conversation
  365. }
  366. i += 1
  367. if i == maxPosts:
  368. break
  369. return personPosts
  370. def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
  371. """Deletes all posts for a person from inbox or outbox
  372. """
  373. if boxname!='inbox' and boxname!='outbox':
  374. return
  375. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  376. for deleteFilename in os.scandir(boxDir):
  377. deleteFilename=deleteFilename.name
  378. filePath = os.path.join(boxDir, deleteFilename)
  379. try:
  380. if os.path.isfile(filePath):
  381. os.unlink(filePath)
  382. elif os.path.isdir(filePath): shutil.rmtree(filePath)
  383. except Exception as e:
  384. print(e)
  385. def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
  386. nickname: str, domain: str,postJsonObject: {}, \
  387. boxname: str) -> str:
  388. """Saves the give json to the give box
  389. Returns the filename
  390. """
  391. if boxname!='inbox' and boxname!='outbox':
  392. return None
  393. originalDomain=domain
  394. if ':' in domain:
  395. domain=domain.split(':')[0]
  396. if not postId:
  397. statusNumber,published = getStatusNumber()
  398. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  399. postJsonObject['id']=postId+'/activity'
  400. if postJsonObject.get('object'):
  401. if isinstance(postJsonObject['object'], dict):
  402. postJsonObject['object']['id']=postId
  403. postJsonObject['object']['atomUri']=postId
  404. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  405. filename=boxDir+'/'+postId.replace('/','#')+'.json'
  406. saveJson(postJsonObject,filename)
  407. return filename
  408. def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
  409. """Writes the post url for hashtags to a file
  410. This allows posts for a hashtag to be quickly looked up
  411. """
  412. if tag['type']!='Hashtag':
  413. return
  414. # create hashtags directory
  415. tagsDir=baseDir+'/tags'
  416. if not os.path.isdir(tagsDir):
  417. os.mkdir(tagsDir)
  418. tagName=tag['name']
  419. tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
  420. tagFile=open(tagsFilename, "a+")
  421. if not tagFile:
  422. return
  423. tagFile.write(newPostId+'\n')
  424. tagFile.close()
  425. def createPostBase(baseDir: str,nickname: str,domain: str,port: int, \
  426. toUrl: str,ccUrl: str,httpPrefix: str,content: str, \
  427. followersOnly: bool,saveToFile: bool,clientToServer: bool, \
  428. attachImageFilename: str,mediaType: str,imageDescription: str, \
  429. useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
  430. inReplyToAtomUri=None,subject=None, \
  431. eventDate=None,eventTime=None,location=None) -> {}:
  432. """Creates a message
  433. """
  434. mentionedRecipients= \
  435. getMentionedPeople(baseDir,httpPrefix,content,domain,False)
  436. tags=[]
  437. hashtagsDict={}
  438. if port:
  439. if port!=80 and port!=443:
  440. if ':' not in domain:
  441. domain=domain+':'+str(port)
  442. # convert content to html
  443. emojisDict={}
  444. # replace 's
  445. content=content.replace("\ufffd\ufffd\ufffds","'s").replace("’","'")
  446. # add tags
  447. content= \
  448. addHtmlTags(baseDir,httpPrefix, \
  449. nickname,domain,content, \
  450. mentionedRecipients, \
  451. hashtagsDict,True)
  452. statusNumber,published = getStatusNumber()
  453. postTo='https://www.w3.org/ns/activitystreams#Public'
  454. postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  455. if followersOnly:
  456. postTo=postCC
  457. postCC=''
  458. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  459. sensitive=False
  460. summary=None
  461. if subject:
  462. summary=subject
  463. sensitive=True
  464. toRecipients=[]
  465. toCC=[]
  466. if toUrl:
  467. if not isinstance(toUrl, str):
  468. print('ERROR: toUrl is not a string')
  469. return None
  470. toRecipients=[toUrl]
  471. # who to send to
  472. if mentionedRecipients:
  473. for mention in mentionedRecipients:
  474. if mention not in toCC:
  475. toCC.append(mention)
  476. # create a list of hashtags
  477. # Only posts which are #Public are searchable by hashtag
  478. if hashtagsDict:
  479. isPublic=False
  480. for recipient in toRecipients:
  481. if recipient.endswith('#Public'):
  482. isPublic=True
  483. break
  484. for tagName,tag in hashtagsDict.items():
  485. tags.append(tag)
  486. if isPublic:
  487. updateHashtagsIndex(baseDir,tag,newPostId)
  488. print('Content tags: '+str(tags))
  489. if inReplyTo and not sensitive:
  490. # locate the post which this is a reply to and check if
  491. # it has a content warning. If it does then reproduce
  492. # the same warning
  493. replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo)
  494. if replyPostFilename:
  495. replyToJson=loadJson(replyPostFilename)
  496. if replyToJson:
  497. if replyToJson.get('object'):
  498. if replyToJson['object'].get('sensitive'):
  499. if replyToJson['object']['sensitive']:
  500. sensitive=True
  501. if replyToJson['object'].get('summary'):
  502. summary=replyToJson['object']['summary']
  503. if eventDate:
  504. eventName=summary
  505. if not eventName:
  506. eventName=content
  507. eventDateStr=eventDate
  508. if eventTime:
  509. if eventTime.endswith('Z'):
  510. eventDateStr=eventDate+'T'+eventTime
  511. else:
  512. eventDateStr=eventDate+'T'+eventTime+':00'+strftime("%z", gmtime())
  513. else:
  514. eventDateStr=eventDate+'T12:00:00Z'
  515. tags.append({
  516. "@context": "https://www.w3.org/ns/activitystreams",
  517. "type": "Event",
  518. "name": eventName,
  519. "startTime": eventDateStr,
  520. "endTime": eventDateStr
  521. })
  522. if location:
  523. tags.append({
  524. "@context": "https://www.w3.org/ns/activitystreams",
  525. "type": "Place",
  526. "name": location
  527. })
  528. postContext=[
  529. 'https://www.w3.org/ns/activitystreams',
  530. {
  531. 'Hashtag': 'as:Hashtag',
  532. 'sensitive': 'as:sensitive',
  533. 'toot': 'http://joinmastodon.org/ns#',
  534. 'votersCount': 'toot:votersCount'
  535. }
  536. ]
  537. if not clientToServer:
  538. actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
  539. # if capabilities have been granted for this actor
  540. # then get the corresponding id
  541. capabilityId=None
  542. capabilityIdList=[]
  543. ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
  544. if ocapFilename:
  545. if os.path.isfile(ocapFilename):
  546. oc=loadJson(ocapFilename)
  547. if oc:
  548. if oc.get('id'):
  549. capabilityIdList=[oc['id']]
  550. newPost = {
  551. "@context": postContext,
  552. 'id': newPostId+'/activity',
  553. 'capability': capabilityIdList,
  554. 'type': 'Create',
  555. 'actor': actorUrl,
  556. 'published': published,
  557. 'to': toRecipients,
  558. 'cc': toCC,
  559. 'object': {
  560. 'id': newPostId,
  561. 'type': 'Note',
  562. 'summary': summary,
  563. 'inReplyTo': inReplyTo,
  564. 'published': published,
  565. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  566. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  567. 'to': toRecipients,
  568. 'cc': toCC,
  569. 'sensitive': sensitive,
  570. 'atomUri': newPostId,
  571. 'inReplyToAtomUri': inReplyToAtomUri,
  572. 'content': content,
  573. 'contentMap': {
  574. 'en': content
  575. },
  576. 'attachment': [],
  577. 'tag': tags,
  578. 'replies': {
  579. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  580. 'type': 'Collection',
  581. 'first': {
  582. 'type': 'CollectionPage',
  583. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  584. 'items': []
  585. }
  586. }
  587. }
  588. }
  589. if attachImageFilename:
  590. newPost['object']= \
  591. attachMedia(baseDir,httpPrefix,domain,port, \
  592. newPost['object'],attachImageFilename, \
  593. mediaType,imageDescription,useBlurhash)
  594. else:
  595. newPost = {
  596. "@context": postContext,
  597. 'id': newPostId,
  598. 'type': 'Note',
  599. 'summary': summary,
  600. 'inReplyTo': inReplyTo,
  601. 'published': published,
  602. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  603. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  604. 'to': toRecipients,
  605. 'cc': toCC,
  606. 'sensitive': sensitive,
  607. 'atomUri': newPostId,
  608. 'inReplyToAtomUri': inReplyToAtomUri,
  609. 'content': content,
  610. 'contentMap': {
  611. 'en': content
  612. },
  613. 'attachment': [],
  614. 'tag': tags,
  615. 'replies': {
  616. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  617. 'type': 'Collection',
  618. 'first': {
  619. 'type': 'CollectionPage',
  620. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  621. 'items': []
  622. }
  623. }
  624. }
  625. if attachImageFilename:
  626. newPost= \
  627. attachMedia(baseDir,httpPrefix,domain,port, \
  628. newPost,attachImageFilename, \
  629. mediaType,imageDescription,useBlurhash)
  630. if ccUrl:
  631. if len(ccUrl)>0:
  632. newPost['cc']=[ccUrl]
  633. if newPost.get('object'):
  634. newPost['object']['cc']=[ccUrl]
  635. # if this is a moderation report then add a status
  636. if isModerationReport:
  637. # add status
  638. if newPost.get('object'):
  639. newPost['object']['moderationStatus']='pending'
  640. else:
  641. newPost['moderationStatus']='pending'
  642. # save to index file
  643. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  644. modFile=open(moderationIndexFile, "a+")
  645. if modFile:
  646. modFile.write(newPostId+'\n')
  647. modFile.close()
  648. if saveToFile:
  649. savePostToBox(baseDir,httpPrefix,newPostId, \
  650. nickname,domain,newPost,'outbox')
  651. return newPost
  652. def outboxMessageCreateWrap(httpPrefix: str, \
  653. nickname: str,domain: str,port: int, \
  654. messageJson: {}) -> {}:
  655. """Wraps a received message in a Create
  656. https://www.w3.org/TR/activitypub/#object-without-create
  657. """
  658. if port:
  659. if port!=80 and port!=443:
  660. if ':' not in domain:
  661. domain=domain+':'+str(port)
  662. statusNumber,published = getStatusNumber()
  663. if messageJson.get('published'):
  664. published = messageJson['published']
  665. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  666. cc=[]
  667. if messageJson.get('cc'):
  668. cc=messageJson['cc']
  669. # TODO
  670. capabilityUrl=[]
  671. newPost = {
  672. "@context": "https://www.w3.org/ns/activitystreams",
  673. 'id': newPostId+'/activity',
  674. 'capability': capabilityUrl,
  675. 'type': 'Create',
  676. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  677. 'published': published,
  678. 'to': messageJson['to'],
  679. 'cc': cc,
  680. 'object': messageJson
  681. }
  682. newPost['object']['id']=newPost['id']
  683. newPost['object']['url']= \
  684. httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
  685. newPost['object']['atomUri']= \
  686. httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  687. return newPost
  688. def postIsAddressedToFollowers(baseDir: str,
  689. nickname: str,domain: str,port: int,httpPrefix: str,
  690. postJsonObject: {}) -> bool:
  691. """Returns true if the given post is addressed to followers of the nickname
  692. """
  693. if port:
  694. if port!=80 and port!=443:
  695. if ':' not in domain:
  696. domain=domain+':'+str(port)
  697. if not postJsonObject.get('object'):
  698. return False
  699. toList=[]
  700. ccList=[]
  701. if postJsonObject['type']!='Update' and \
  702. isinstance(postJsonObject['object'], dict):
  703. if postJsonObject['object'].get('to'):
  704. toList=postJsonObject['object']['to']
  705. if postJsonObject['object'].get('cc'):
  706. ccList=postJsonObject['object']['cc']
  707. else:
  708. if postJsonObject.get('to'):
  709. toList=postJsonObject['to']
  710. if postJsonObject.get('cc'):
  711. ccList=postJsonObject['cc']
  712. followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  713. # does the followers url exist in 'to' or 'cc' lists?
  714. addressedToFollowers=False
  715. if followersUrl in toList:
  716. addressedToFollowers=True
  717. elif followersUrl in ccList:
  718. addressedToFollowers=True
  719. return addressedToFollowers
  720. def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
  721. """Returns true if the given post is addressed to public
  722. """
  723. if not postJsonObject.get('object'):
  724. return False
  725. if not postJsonObject['object'].get('to'):
  726. return False
  727. publicUrl='https://www.w3.org/ns/activitystreams#Public'
  728. # does the public url exist in 'to' or 'cc' lists?
  729. addressedToPublic=False
  730. if publicUrl in postJsonObject['object']['to']:
  731. addressedToPublic=True
  732. if not addressedToPublic:
  733. if not postJsonObject['object'].get('cc'):
  734. return False
  735. if publicUrl in postJsonObject['object']['cc']:
  736. addressedToPublic=True
  737. return addressedToPublic
  738. def createPublicPost(baseDir: str,
  739. nickname: str,domain: str,port: int,httpPrefix: str, \
  740. content: str,followersOnly: bool,saveToFile: bool,
  741. clientToServer: bool,\
  742. attachImageFilename: str,mediaType: str, \
  743. imageDescription: str,useBlurhash: bool, \
  744. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  745. eventDate=None,eventTime=None,location=None) -> {}:
  746. """Public post
  747. """
  748. domainFull=domain
  749. if port:
  750. if port!=80 and port!=443:
  751. if ':' not in domain:
  752. domainFull=domain+':'+str(port)
  753. return createPostBase(baseDir,nickname,domain,port, \
  754. 'https://www.w3.org/ns/activitystreams#Public', \
  755. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  756. httpPrefix,content,followersOnly,saveToFile, \
  757. clientToServer, \
  758. attachImageFilename,mediaType, \
  759. imageDescription,useBlurhash, \
  760. False,inReplyTo,inReplyToAtomUri,subject, \
  761. eventDate,eventTime,location)
  762. def createUnlistedPost(baseDir: str,
  763. nickname: str,domain: str,port: int,httpPrefix: str, \
  764. content: str,followersOnly: bool,saveToFile: bool,
  765. clientToServer: bool,\
  766. attachImageFilename: str,mediaType: str, \
  767. imageDescription: str,useBlurhash: bool, \
  768. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  769. eventDate=None,eventTime=None,location=None) -> {}:
  770. """Unlisted post. This has the #Public and followers links inverted.
  771. """
  772. domainFull=domain
  773. if port:
  774. if port!=80 and port!=443:
  775. if ':' not in domain:
  776. domainFull=domain+':'+str(port)
  777. return createPostBase(baseDir,nickname,domain,port, \
  778. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  779. 'https://www.w3.org/ns/activitystreams#Public', \
  780. httpPrefix,content,followersOnly,saveToFile, \
  781. clientToServer, \
  782. attachImageFilename,mediaType, \
  783. imageDescription,useBlurhash, \
  784. False,inReplyTo, inReplyToAtomUri, subject, \
  785. eventDate,eventTime,location)
  786. def createFollowersOnlyPost(baseDir: str,
  787. nickname: str,domain: str,port: int,httpPrefix: str, \
  788. content: str,followersOnly: bool,saveToFile: bool,
  789. clientToServer: bool,\
  790. attachImageFilename: str,mediaType: str, \
  791. imageDescription: str,useBlurhash: bool, \
  792. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  793. eventDate=None,eventTime=None,location=None) -> {}:
  794. """Followers only post
  795. """
  796. domainFull=domain
  797. if port:
  798. if port!=80 and port!=443:
  799. if ':' not in domain:
  800. domainFull=domain+':'+str(port)
  801. return createPostBase(baseDir,nickname,domain,port, \
  802. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  803. None,
  804. httpPrefix,content,followersOnly,saveToFile, \
  805. clientToServer, \
  806. attachImageFilename,mediaType, \
  807. imageDescription,useBlurhash, \
  808. False,inReplyTo, inReplyToAtomUri, subject, \
  809. eventDate,eventTime,location)
  810. def getMentionedPeople(baseDir: str,httpPrefix: str, \
  811. content: str,domain: str,debug: bool) -> []:
  812. """Extracts a list of mentioned actors from the given message content
  813. """
  814. if '@' not in content:
  815. return None
  816. mentions=[]
  817. words=content.split(' ')
  818. for wrd in words:
  819. if wrd.startswith('@'):
  820. handle=wrd[1:]
  821. if debug:
  822. print('DEBUG: mentioned handle '+handle)
  823. if '@' not in handle:
  824. handle=handle+'@'+domain
  825. if not os.path.isdir(baseDir+'/accounts/'+handle):
  826. continue
  827. else:
  828. externalDomain=handle.split('@')[1]
  829. if not ('.' in externalDomain or externalDomain=='localhost'):
  830. continue
  831. mentionedNickname=handle.split('@')[0]
  832. mentionedDomain=handle.split('@')[1].strip('\n')
  833. if ':' in mentionedDomain:
  834. mentionedDomain=mentionedDomain.split(':')[0]
  835. if not validNickname(mentionedDomain,mentionedNickname):
  836. continue
  837. actor=httpPrefix+'://'+handle.split('@')[1]+'/users/'+mentionedNickname
  838. mentions.append(actor)
  839. return mentions
  840. def createDirectMessagePost(baseDir: str,
  841. nickname: str,domain: str,port: int,httpPrefix: str, \
  842. content: str,followersOnly: bool,saveToFile: bool,
  843. clientToServer: bool,\
  844. attachImageFilename: str,mediaType: str, \
  845. imageDescription: str,useBlurhash: bool, \
  846. inReplyTo=None,inReplyToAtomUri=None,subject=None,debug=False, \
  847. eventDate=None,eventTime=None,location=None) -> {}:
  848. """Direct Message post
  849. """
  850. mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
  851. if debug:
  852. print('mentionedPeople: '+str(mentionedPeople))
  853. if not mentionedPeople:
  854. return None
  855. postTo=None
  856. postCc=None
  857. messageJson= \
  858. createPostBase(baseDir,nickname,domain,port, \
  859. postTo,postCc, \
  860. httpPrefix,content,followersOnly,saveToFile, \
  861. clientToServer, \
  862. attachImageFilename,mediaType, \
  863. imageDescription,useBlurhash, \
  864. False,inReplyTo,inReplyToAtomUri,subject, \
  865. eventDate,eventTime,location)
  866. # mentioned recipients go into To rather than Cc
  867. messageJson['to']=messageJson['object']['cc']
  868. messageJson['object']['to']=messageJson['to']
  869. messageJson['cc']=[]
  870. messageJson['object']['cc']=[]
  871. return messageJson
  872. def createReportPost(baseDir: str,
  873. nickname: str, domain: str, port: int,httpPrefix: str, \
  874. content: str, followersOnly: bool, saveToFile: bool,
  875. clientToServer: bool,\
  876. attachImageFilename: str,mediaType: str, \
  877. imageDescription: str,useBlurhash: bool, \
  878. debug: bool,subject=None) -> {}:
  879. """Send a report to moderators
  880. """
  881. domainFull=domain
  882. if port:
  883. if port!=80 and port!=443:
  884. if ':' not in domain:
  885. domainFull=domain+':'+str(port)
  886. # add a title to distinguish moderation reports from other posts
  887. reportTitle='Moderation Report'
  888. if not subject:
  889. subject=reportTitle
  890. else:
  891. if not subject.startswith(reportTitle):
  892. subject=reportTitle+': '+subject
  893. # create the list of moderators from the moderators file
  894. moderatorsList=[]
  895. moderatorsFile=baseDir+'/accounts/moderators.txt'
  896. if os.path.isfile(moderatorsFile):
  897. with open (moderatorsFile, "r") as fileHandler:
  898. for line in fileHandler:
  899. line=line.strip('\n')
  900. if line.startswith('#'):
  901. continue
  902. if line.startswith('/users/'):
  903. line=line.replace('users','')
  904. if line.startswith('@'):
  905. line=line[1:]
  906. if '@' in line:
  907. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
  908. if moderatorActor not in moderatorList:
  909. moderatorsList.append(moderatorActor)
  910. continue
  911. if line.startswith('http') or line.startswith('dat'):
  912. # must be a local address - no remote moderators
  913. if '://'+domainFull+'/' in line:
  914. if line not in moderatorsList:
  915. moderatorsList.append(line)
  916. else:
  917. if '/' not in line:
  918. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
  919. if moderatorActor not in moderatorsList:
  920. moderatorsList.append(moderatorActor)
  921. if len(moderatorsList)==0:
  922. # if there are no moderators then the admin becomes the moderator
  923. adminNickname=getConfigParam(baseDir,'admin')
  924. if adminNickname:
  925. moderatorsList.append(httpPrefix+'://'+domainFull+'/users/'+adminNickname)
  926. if not moderatorsList:
  927. return None
  928. if debug:
  929. print('DEBUG: Sending report to moderators')
  930. print(str(moderatorsList))
  931. postTo=moderatorsList
  932. postCc=None
  933. postJsonObject=None
  934. for toUrl in postTo:
  935. # who is this report going to?
  936. toNickname=toUrl.split('/users/')[1]
  937. handle=toNickname+'@'+domain
  938. postJsonObject= \
  939. createPostBase(baseDir,nickname,domain,port, \
  940. toUrl,postCc, \
  941. httpPrefix,content,followersOnly,saveToFile, \
  942. clientToServer, \
  943. attachImageFilename,mediaType, \
  944. imageDescription,useBlurhash, \
  945. True,None,None,subject, \
  946. None,None,None)
  947. if not postJsonObject:
  948. continue
  949. # update the inbox index with the report filename
  950. #indexFilename=baseDir+'/accounts/'+handle+'/inbox.index'
  951. #indexEntry=postJsonObject['id'].replace('/activity','').replace('/','#')+'.json'
  952. #if indexEntry not in open(indexFilename).read():
  953. # try:
  954. # with open(indexFilename, 'a+') as fp:
  955. # fp.write(indexEntry)
  956. # except:
  957. # pass
  958. # save a notification file so that the moderator
  959. # knows something new has appeared
  960. newReportFile=baseDir+'/accounts/'+handle+'/.newReport'
  961. if os.path.isfile(newReportFile):
  962. continue
  963. try:
  964. with open(newReportFile, 'w') as fp:
  965. fp.write(toUrl+'/moderation')
  966. except:
  967. pass
  968. return postJsonObject
  969. def threadSendPost(session,postJsonStr: str,federationList: [],\
  970. inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
  971. debug :bool) -> None:
  972. """Sends a with retries
  973. """
  974. tries=0
  975. sendIntervalSec=30
  976. for attempt in range(20):
  977. postResult=None
  978. unauthorized=False
  979. try:
  980. postResult,unauthorized = \
  981. postJsonString(session,postJsonStr,federationList, \
  982. inboxUrl,signatureHeaderJson, \
  983. "inbox:write",debug)
  984. except Exception as e:
  985. print('ERROR: postJsonString failed '+str(e))
  986. if unauthorized==True:
  987. print(postJsonStr)
  988. print('threadSendPost: Post is unauthorized')
  989. break
  990. if postResult:
  991. logStr='Success on try '+str(tries)+': '+postJsonStr
  992. else:
  993. logStr='Retry '+str(tries)+': '+postJsonStr
  994. postLog.append(logStr)
  995. # keep the length of the log finite
  996. # Don't accumulate massive files on systems with limited resources
  997. while len(postLog)>16:
  998. postLog.pop(0)
  999. if debug:
  1000. # save the log file
  1001. postLogFilename=baseDir+'/post.log'
  1002. with open(postLogFilename, "a+") as logFile:
  1003. logFile.write(logStr+'\n')
  1004. if postResult:
  1005. if debug:
  1006. print('DEBUG: successful json post to '+inboxUrl)
  1007. # our work here is done
  1008. break
  1009. if debug:
  1010. print(postJsonStr)
  1011. print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
  1012. str(sendIntervalSec)+' seconds.')
  1013. time.sleep(sendIntervalSec)
  1014. tries+=1
  1015. def sendPost(projectVersion: str, \
  1016. session,baseDir: str,nickname: str, domain: str, port: int, \
  1017. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1018. httpPrefix: str, content: str, followersOnly: bool, \
  1019. saveToFile: bool, clientToServer: bool, \
  1020. attachImageFilename: str,mediaType: str, \
  1021. imageDescription: str,useBlurhash: bool, \
  1022. federationList: [],\
  1023. sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
  1024. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  1025. """Post to another inbox
  1026. """
  1027. withDigest=True
  1028. if toNickname=='inbox':
  1029. # shared inbox actor on @domain@domain
  1030. toNickname=toDomain
  1031. toDomainOriginal=toDomain
  1032. if toPort:
  1033. if toPort!=80 and toPort!=443:
  1034. if ':' not in toDomain:
  1035. toDomain=toDomain+':'+str(toPort)
  1036. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  1037. # lookup the inbox for the To handle
  1038. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1039. domain,projectVersion)
  1040. if not wfRequest:
  1041. return 1
  1042. if not clientToServer:
  1043. postToBox='inbox'
  1044. else:
  1045. postToBox='outbox'
  1046. # get the actor inbox for the To handle
  1047. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1048. getPersonBox(baseDir,session,wfRequest,personCache, \
  1049. projectVersion,httpPrefix, \
  1050. nickname,domain,postToBox)
  1051. # If there are more than one followers on the target domain
  1052. # then send to the shared inbox indead of the individual inbox
  1053. if nickname=='capabilities':
  1054. inboxUrl=capabilityAcquisition
  1055. if not capabilityAcquisition:
  1056. return 2
  1057. if not inboxUrl:
  1058. return 3
  1059. if not pubKey:
  1060. return 4
  1061. if not toPersonId:
  1062. return 5
  1063. # sharedInbox and capabilities are optional
  1064. postJsonObject = \
  1065. createPostBase(baseDir,nickname,domain,port, \
  1066. toPersonId,cc,httpPrefix,content, \
  1067. followersOnly,saveToFile,clientToServer, \
  1068. attachImageFilename,mediaType, \
  1069. imageDescription,useBlurhash, \
  1070. False,inReplyTo,inReplyToAtomUri,subject, \
  1071. None,None,None)
  1072. # get the senders private key
  1073. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
  1074. if len(privateKeyPem)==0:
  1075. return 6
  1076. if toDomain not in inboxUrl:
  1077. return 7
  1078. postPath=inboxUrl.split(toDomain,1)[1]
  1079. # convert json to string so that there are no
  1080. # subsequent conversions after creating message body digest
  1081. postJsonStr=json.dumps(postJsonObject)
  1082. # construct the http header, including the message body digest
  1083. signatureHeaderJson = \
  1084. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1085. toDomain,toPort, \
  1086. postPath,httpPrefix,withDigest,postJsonStr)
  1087. # Keep the number of threads being used small
  1088. while len(sendThreads)>1000:
  1089. print('WARN: Maximum threads reached - killing send thread')
  1090. sendThreads[0].kill()
  1091. sendThreads.pop(0)
  1092. print('WARN: thread killed')
  1093. thr = threadWithTrace(target=threadSendPost,args=(session, \
  1094. postJsonStr, \
  1095. federationList, \
  1096. inboxUrl,baseDir, \
  1097. signatureHeaderJson.copy(), \
  1098. postLog,
  1099. debug),daemon=True)
  1100. sendThreads.append(thr)
  1101. thr.start()
  1102. return 0
  1103. def sendPostViaServer(projectVersion: str, \
  1104. baseDir: str,session,fromNickname: str,password: str, \
  1105. fromDomain: str, fromPort: int, \
  1106. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1107. httpPrefix: str, content: str, followersOnly: bool, \
  1108. attachImageFilename: str,mediaType: str, \
  1109. imageDescription: str,useBlurhash: bool, \
  1110. cachedWebfingers: {},personCache: {}, \
  1111. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  1112. """Send a post via a proxy (c2s)
  1113. """
  1114. if not session:
  1115. print('WARN: No session for sendPostViaServer')
  1116. return 6
  1117. withDigest=True
  1118. if toPort:
  1119. if toPort!=80 and toPort!=443:
  1120. if ':' not in fromDomain:
  1121. fromDomain=fromDomain+':'+str(fromPort)
  1122. handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
  1123. # lookup the inbox for the To handle
  1124. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1125. fromDomain,projectVersion)
  1126. if not wfRequest:
  1127. if debug:
  1128. print('DEBUG: webfinger failed for '+handle)
  1129. return 1
  1130. postToBox='outbox'
  1131. # get the actor inbox for the To handle
  1132. inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1133. getPersonBox(baseDir,session,wfRequest,personCache, \
  1134. projectVersion,httpPrefix,fromNickname, \
  1135. fromDomain,postToBox)
  1136. if not inboxUrl:
  1137. if debug:
  1138. print('DEBUG: No '+postToBox+' was found for '+handle)
  1139. return 3
  1140. if not fromPersonId:
  1141. if debug:
  1142. print('DEBUG: No actor was found for '+handle)
  1143. return 4
  1144. # Get the json for the c2s post, not saving anything to file
  1145. # Note that baseDir is set to None
  1146. saveToFile=False
  1147. clientToServer=True
  1148. if toDomain.lower().endswith('public'):
  1149. toPersonId='https://www.w3.org/ns/activitystreams#Public'
  1150. fromDomainFull=fromDomain
  1151. if fromPort:
  1152. if fromPort!=80 and fromPort!=443:
  1153. if ':' not in fromDomain:
  1154. fromDomainFull=fromDomain+':'+str(fromPort)
  1155. cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1156. else:
  1157. if toDomain.lower().endswith('followers') or \
  1158. toDomain.lower().endswith('followersonly'):
  1159. toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1160. else:
  1161. toDomainFull=toDomain
  1162. if toPort:
  1163. if toPort!=80 and toPort!=443:
  1164. if ':' not in toDomain:
  1165. toDomainFull=toDomain+':'+str(toPort)
  1166. toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
  1167. postJsonObject = \
  1168. createPostBase(baseDir, \
  1169. fromNickname,fromDomain,fromPort, \
  1170. toPersonId,cc,httpPrefix,content, \
  1171. followersOnly,saveToFile,clientToServer, \
  1172. attachImageFilename,mediaType, \
  1173. imageDescription,useBlurhash, \
  1174. False,inReplyTo,inReplyToAtomUri,subject, \
  1175. None,None,None)
  1176. authHeader=createBasicAuthHeader(fromNickname,password)
  1177. if attachImageFilename:
  1178. headers = {'host': fromDomain, \
  1179. 'Authorization': authHeader}
  1180. postResult = \
  1181. postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
  1182. #if not postResult:
  1183. # if debug:
  1184. # print('DEBUG: Failed to upload image')
  1185. # return 9
  1186. headers = {'host': fromDomain, \
  1187. 'Content-type': 'application/json', \
  1188. 'Authorization': authHeader}
  1189. postResult = \
  1190. postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write",debug)
  1191. #if not postResult:
  1192. # if debug:
  1193. # print('DEBUG: POST failed for c2s to '+inboxUrl)
  1194. # return 5
  1195. if debug:
  1196. print('DEBUG: c2s POST success')
  1197. return 0
  1198. def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
  1199. """Returns a dictionary with followers grouped by domain
  1200. """
  1201. handle=nickname+'@'+domain
  1202. followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
  1203. if not os.path.isfile(followersFilename):
  1204. return None
  1205. grouped={}
  1206. with open(followersFilename, "r") as f:
  1207. for followerHandle in f:
  1208. if '@' in followerHandle:
  1209. fHandle=followerHandle.strip().replace('\n','')
  1210. followerDomain=fHandle.split('@')[1]
  1211. if not grouped.get(followerDomain):
  1212. grouped[followerDomain]=[fHandle]
  1213. else:
  1214. grouped[followerDomain].append(fHandle)
  1215. return grouped
  1216. def addFollowersToPublicPost(postJsonObject: {}) -> None:
  1217. """Adds followers entry to cc if it doesn't exist
  1218. """
  1219. if not postJsonObject.get('actor'):
  1220. return
  1221. if isinstance(postJsonObject['object'], str):
  1222. if not postJsonObject.get('to'):
  1223. return
  1224. if len(postJsonObject['to'])>1:
  1225. return
  1226. if len(postJsonObject['to'])==0:
  1227. return
  1228. if not postJsonObject['to'][0].endswith('#Public'):
  1229. return
  1230. if postJsonObject.get('cc'):
  1231. return
  1232. postJsonObject['cc']=postJsonObject['actor']+'/followers'
  1233. elif isinstance(postJsonObject['object'], dict):
  1234. if not postJsonObject['object'].get('to'):
  1235. return
  1236. if len(postJsonObject['object']['to'])>1:
  1237. return
  1238. if len(postJsonObject['object']['to'])==0:
  1239. return
  1240. if not postJsonObject['object']['to'][0].endswith('#Public'):
  1241. return
  1242. if postJsonObject['object'].get('cc'):
  1243. return
  1244. postJsonObject['object']['cc']=postJsonObject['actor']+'/followers'
  1245. def sendSignedJson(postJsonObject: {},session,baseDir: str, \
  1246. nickname: str, domain: str, port: int, \
  1247. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1248. httpPrefix: str, saveToFile: bool, clientToServer: bool, \
  1249. federationList: [], \
  1250. sendThreads: [], postLog: [], cachedWebfingers: {}, \
  1251. personCache: {}, debug: bool,projectVersion: str) -> int:
  1252. """Sends a signed json object to an inbox/outbox
  1253. """
  1254. if debug:
  1255. print('DEBUG: sendSignedJson start')
  1256. if not session:
  1257. print('WARN: No session specified for sendSignedJson')
  1258. return 8
  1259. withDigest=True
  1260. sharedInbox=False
  1261. if toNickname=='inbox':
  1262. # shared inbox actor on @domain@domain
  1263. toNickname=toDomain
  1264. sharedInbox=True
  1265. toDomainOriginal=toDomain
  1266. if toPort:
  1267. if toPort!=80 and toPort!=443:
  1268. if ':' not in toDomain:
  1269. toDomain=toDomain+':'+str(toPort)
  1270. handleBase=httpPrefix+'://'+toDomain+'/@'
  1271. if toNickname:
  1272. handle=handleBase+toNickname
  1273. else:
  1274. singleUserInstanceNickname='dev'
  1275. handle=handleBase+singleUserInstanceNickname
  1276. if debug:
  1277. print('DEBUG: handle - '+handle+' toPort '+str(toPort))
  1278. # lookup the inbox for the To handle
  1279. wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1280. domain,projectVersion)
  1281. if not wfRequest:
  1282. if debug:
  1283. print('DEBUG: webfinger for '+handle+' failed')
  1284. return 1
  1285. if wfRequest.get('errors'):
  1286. if debug:
  1287. print('DEBUG: webfinger for '+handle+' failed with errors '+str(wfRequest['errors']))
  1288. if not clientToServer:
  1289. postToBox='inbox'
  1290. else:
  1291. postToBox='outbox'
  1292. # get the actor inbox/outbox/capabilities for the To handle
  1293. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
  1294. getPersonBox(baseDir,session,wfRequest,personCache, \
  1295. projectVersion,httpPrefix,nickname,domain,postToBox)
  1296. if nickname=='capabilities':
  1297. inboxUrl=capabilityAcquisition
  1298. if not capabilityAcquisition:
  1299. return 2
  1300. else:
  1301. print("inboxUrl: "+str(inboxUrl))
  1302. print("toPersonId: "+str(toPersonId))
  1303. print("sharedInboxUrl: "+str(sharedInboxUrl))
  1304. if inboxUrl:
  1305. if inboxUrl.endswith('/actor/inbox'):
  1306. inboxUrl=sharedInboxUrl
  1307. if not inboxUrl:
  1308. if debug:
  1309. print('DEBUG: missing inboxUrl')
  1310. return 3
  1311. if debug:
  1312. print('DEBUG: Sending to endpoint '+inboxUrl)
  1313. if not pubKey:
  1314. if debug:
  1315. print('DEBUG: missing pubkey')
  1316. return 4
  1317. if not toPersonId:
  1318. if debug:
  1319. print('DEBUG: missing personId')
  1320. return 5
  1321. # sharedInbox and capabilities are optional
  1322. # get the senders private key
  1323. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
  1324. if len(privateKeyPem)==0:
  1325. if debug:
  1326. print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
  1327. return 6
  1328. if toDomain not in inboxUrl:
  1329. if debug:
  1330. print('DEBUG: '+toDomain+' is not in '+inboxUrl)
  1331. return 7
  1332. postPath=inboxUrl.split(toDomain,1)[1]
  1333. addFollowersToPublicPost(postJsonObject)
  1334. # convert json to string so that there are no
  1335. # subsequent conversions after creating message body digest
  1336. postJsonStr=json.dumps(postJsonObject)
  1337. # construct the http header, including the message body digest
  1338. signatureHeaderJson = \
  1339. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1340. toDomain,toPort, \
  1341. postPath,httpPrefix,withDigest,postJsonStr)
  1342. # Keep the number of threads being used small
  1343. while len(sendThreads)>1000:
  1344. print('WARN: Maximum threads reached - killing send thread')
  1345. sendThreads[0].kill()
  1346. sendThreads.pop(0)
  1347. print('WARN: thread killed')
  1348. if debug:
  1349. print('DEBUG: starting thread to send post')
  1350. pprint(postJsonObject)
  1351. thr = threadWithTrace(target=threadSendPost, \
  1352. args=(session, \
  1353. postJsonStr, \
  1354. federationList, \
  1355. inboxUrl,baseDir, \
  1356. signatureHeaderJson.copy(), \
  1357. postLog,
  1358. debug),daemon=True)
  1359. sendThreads.append(thr)
  1360. #thr.start()
  1361. return 0
  1362. def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
  1363. """The Follow activity doesn't have a 'to' field and so one
  1364. needs to be added so that activity distribution happens in a consistent way
  1365. Returns true if a 'to' field exists or was added
  1366. """
  1367. if postJsonObject.get('to'):
  1368. return postJsonObject,True
  1369. if debug:
  1370. pprint(postJsonObject)
  1371. print('DEBUG: no "to" field when sending to named addresses 2')
  1372. isSameType=False
  1373. toFieldAdded=False
  1374. if postJsonObject.get('object'):
  1375. if isinstance(postJsonObject['object'], str):
  1376. if postJsonObject.get('type'):
  1377. if postJsonObject['type']==activityType:
  1378. isSameType=True
  1379. if debug:
  1380. print('DEBUG: "to" field assigned to Follow')
  1381. toAddress=postJsonObject['object']
  1382. if '/statuses/' in toAddress:
  1383. toAddress=toAddress.split('/statuses/')[0]
  1384. postJsonObject['to']=[toAddress]
  1385. toFieldAdded=True
  1386. elif isinstance(postJsonObject['object'], dict):
  1387. if postJsonObject['object'].get('type'):
  1388. if postJsonObject['object']['type']==activityType:
  1389. isSameType=True
  1390. if isinstance(postJsonObject['object']['object'], str):
  1391. if debug:
  1392. print('DEBUG: "to" field assigned to Follow')
  1393. toAddress=postJsonObject['object']['object']
  1394. if '/statuses/' in toAddress:
  1395. toAddress=toAddress.split('/statuses/')[0]
  1396. postJsonObject['object']['to']=[toAddress]
  1397. postJsonObject['to']=[postJsonObject['object']['object']]
  1398. toFieldAdded=True
  1399. if not isSameType:
  1400. return postJsonObject,True
  1401. if toFieldAdded:
  1402. return postJsonObject,True
  1403. return postJsonObject,False
  1404. def sendToNamedAddresses(session,baseDir: str, \
  1405. nickname: str, domain: str, port: int, \
  1406. httpPrefix: str,federationList: [], \
  1407. sendThreads: [],postLog: [], \
  1408. cachedWebfingers: {},personCache: {}, \
  1409. postJsonObject: {},debug: bool, \
  1410. projectVersion: str) -> None:
  1411. """sends a post to the specific named addresses in to/cc
  1412. """
  1413. if not session:
  1414. print('WARN: No session for sendToNamedAddresses')
  1415. return
  1416. if not postJsonObject.get('object'):
  1417. return
  1418. if isinstance(postJsonObject['object'], dict):
  1419. isProfileUpdate=False
  1420. # for actor updates there is no 'to' within the object
  1421. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1422. if postJsonObject['type']=='Update' and \
  1423. (postJsonObject['object']['type']=='Person' or \
  1424. postJsonObject['object']['type']=='Application' or \
  1425. postJsonObject['object']['type']=='Group' or \
  1426. postJsonObject['object']['type']=='Service'):
  1427. # use the original object, which has a 'to'
  1428. recipientsObject=postJsonObject
  1429. isProfileUpdate=True
  1430. if not isProfileUpdate:
  1431. if not postJsonObject['object'].get('to'):
  1432. if debug:
  1433. pprint(postJsonObject)
  1434. print('DEBUG: no "to" field when sending to named addresses')
  1435. if postJsonObject['object'].get('type'):
  1436. if postJsonObject['object']['type']=='Follow':
  1437. if isinstance(postJsonObject['object']['object'], str):
  1438. if debug:
  1439. print('DEBUG: "to" field assigned to Follow')
  1440. postJsonObject['object']['to']=[postJsonObject['object']['object']]
  1441. if not postJsonObject['object'].get('to'):
  1442. return
  1443. recipientsObject=postJsonObject['object']
  1444. else:
  1445. postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
  1446. if not fieldAdded:
  1447. return
  1448. postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
  1449. if not fieldAdded:
  1450. return
  1451. recipientsObject=postJsonObject
  1452. recipients=[]
  1453. recipientType=['to','cc']
  1454. for rType in recipientType:
  1455. if not recipientsObject.get(rType):
  1456. continue
  1457. if isinstance(recipientsObject[rType], list):
  1458. if debug:
  1459. pprint(recipientsObject)
  1460. print('recipientsObject: '+str(recipientsObject[rType]))
  1461. for address in recipientsObject[rType]:
  1462. if not address:
  1463. continue
  1464. if '/' not in address:
  1465. continue
  1466. if address.endswith('#Public'):
  1467. continue
  1468. if address.endswith('/followers'):
  1469. continue
  1470. recipients.append(address)
  1471. elif isinstance(recipientsObject[rType], str):
  1472. address=recipientsObject[rType]
  1473. if address:
  1474. if '/' in address:
  1475. if address.endswith('#Public'):
  1476. continue
  1477. if address.endswith('/followers'):
  1478. continue
  1479. recipients.append(address)
  1480. if not recipients:
  1481. if debug:
  1482. print('DEBUG: no individual recipients')
  1483. return
  1484. if debug:
  1485. print('DEBUG: Sending individually addressed posts: '+str(recipients))
  1486. # this is after the message has arrived at the server
  1487. clientToServer=False
  1488. for address in recipients:
  1489. toNickname=getNicknameFromActor(address)
  1490. if not toNickname:
  1491. continue
  1492. toDomain,toPort=getDomainFromActor(address)
  1493. if not toDomain:
  1494. continue
  1495. if debug:
  1496. domainFull=domain
  1497. if port:
  1498. if port!=80 and port!=443:
  1499. if ':' not in domain:
  1500. domainFull=domain+':'+str(port)
  1501. toDomainFull=toDomain
  1502. if toPort:
  1503. if toPort!=80 and toPort!=443:
  1504. if ':' not in toDomain:
  1505. toDomainFull=toDomain+':'+str(toPort)
  1506. print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
  1507. cc=[]
  1508. sendSignedJson(postJsonObject,session,baseDir, \
  1509. nickname,domain,port, \
  1510. toNickname,toDomain,toPort, \
  1511. cc,httpPrefix,True,clientToServer, \
  1512. federationList, \
  1513. sendThreads,postLog,cachedWebfingers, \
  1514. personCache,debug,projectVersion)
  1515. def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
  1516. """Returns true if the given domain has a shared inbox
  1517. """
  1518. wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
  1519. None,__version__)
  1520. if wfRequest:
  1521. if not wfRequest.get('errors'):
  1522. return True
  1523. return False
  1524. def sendToFollowers(session,baseDir: str, \
  1525. nickname: str, domain: str, port: int, \
  1526. httpPrefix: str,federationList: [], \
  1527. sendThreads: [],postLog: [], \
  1528. cachedWebfingers: {},personCache: {}, \
  1529. postJsonObject: {},debug: bool, \
  1530. projectVersion: str) -> None:
  1531. """sends a post to the followers of the given nickname
  1532. """
  1533. print('sendToFollowers')
  1534. if not session:
  1535. print('WARN: No session for sendToFollowers')
  1536. return
  1537. if not postIsAddressedToFollowers(baseDir,nickname,domain, \
  1538. port,httpPrefix,postJsonObject):
  1539. if debug:
  1540. print('Post is not addressed to followers')
  1541. return
  1542. print('Post is addressed to followers')
  1543. grouped=groupFollowersByDomain(baseDir,nickname,domain)
  1544. if not grouped:
  1545. if debug:
  1546. print('Post to followers did not resolve any domains')
  1547. return
  1548. print('Post to followers resolved domains')
  1549. print(str(grouped))
  1550. # this is after the message has arrived at the server
  1551. clientToServer=False
  1552. # for each instance
  1553. for followerDomain,followerHandles in grouped.items():
  1554. if debug:
  1555. print('DEBUG: follower handles for '+followerDomain)
  1556. pprint(followerHandles)
  1557. withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
  1558. if debug:
  1559. if withSharedInbox:
  1560. print(followerDomain+' has shared inbox')
  1561. else:
  1562. print(followerDomain+' does not have a shared inbox')
  1563. toPort=port
  1564. index=0
  1565. toDomain=followerHandles[index].split('@')[1]
  1566. if ':' in toDomain:
  1567. toPort=toDomain.split(':')[1]
  1568. toDomain=toDomain.split(':')[0]
  1569. cc=''
  1570. if withSharedInbox:
  1571. toNickname=followerHandles[index].split('@')[0]
  1572. # if there are more than one followers on the domain
  1573. # then send the post to the shared inbox
  1574. if len(followerHandles)>1:
  1575. toNickname='inbox'
  1576. if toNickname!='inbox' and postJsonObject.get('type'):
  1577. if postJsonObject['type']=='Update':
  1578. if postJsonObject.get('object'):
  1579. if isinstance(postJsonObject['object'], dict):
  1580. if postJsonObject['object'].get('type'):
  1581. if postJsonObject['object']['type']=='Person' or \
  1582. postJsonObject['object']['type']=='Application' or \
  1583. postJsonObject['object']['type']=='Group' or \
  1584. postJsonObject['object']['type']=='Service':
  1585. print('Sending profile update to shared inbox of '+toDomain)
  1586. toNickname='inbox'
  1587. if debug:
  1588. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1589. sendSignedJson(postJsonObject,session,baseDir, \
  1590. nickname,domain,port, \
  1591. toNickname,toDomain,toPort, \
  1592. cc,httpPrefix,True,clientToServer, \
  1593. federationList, \
  1594. sendThreads,postLog,cachedWebfingers, \
  1595. personCache,debug,projectVersion)
  1596. else:
  1597. # send to individual followers without using a shared inbox
  1598. for handle in followerHandles:
  1599. if debug:
  1600. print('DEBUG: Sending to '+handle)
  1601. toNickname=handle.split('@')[0]
  1602. if debug:
  1603. if postJsonObject['type']!='Update':
  1604. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1605. else:
  1606. print('DEBUG: Sending profile update from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1607. sendSignedJson(postJsonObject,session,baseDir, \
  1608. nickname,domain,port, \
  1609. toNickname,toDomain,toPort, \
  1610. cc,httpPrefix,True,clientToServer, \
  1611. federationList, \
  1612. sendThreads,postLog,cachedWebfingers, \
  1613. personCache,debug,projectVersion)
  1614. time.sleep(4)
  1615. if debug:
  1616. print('DEBUG: End of sendToFollowers')
  1617. def sendToFollowersThread(session,baseDir: str, \
  1618. nickname: str, domain: str, port: int, \
  1619. httpPrefix: str,federationList: [], \
  1620. sendThreads: [],postLog: [], \
  1621. cachedWebfingers: {},personCache: {}, \
  1622. postJsonObject: {},debug: bool, \
  1623. projectVersion: str):
  1624. """Returns a thread used to send a post to followers
  1625. """
  1626. sendThread= \
  1627. threadWithTrace(target=sendToFollowers, \
  1628. args=(session,baseDir, \
  1629. nickname,domain,port, \
  1630. httpPrefix,federationList, \
  1631. sendThreads,postLog, \
  1632. cachedWebfingers,personCache, \
  1633. postJsonObject.copy(),debug, \
  1634. projectVersion),daemon=True)
  1635. sendThread.start()
  1636. return sendThread
  1637. def createInbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1638. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1639. return createBoxIndexed(session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
  1640. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1641. def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1642. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1643. return createBoxIndexed(session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
  1644. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1645. def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1646. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1647. return createBoxIndexed(session,baseDir,'dm',nickname,domain,port,httpPrefix, \
  1648. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1649. def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1650. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1651. return createBoxIndexed(session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
  1652. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1653. def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1654. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1655. return createBoxIndexed(session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
  1656. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1657. def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1658. itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
  1659. return createBoxIndexed(session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
  1660. itemsPerPage,headerOnly,authorized,False,pageNumber)
  1661. def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1662. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1663. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1664. boxname='moderation'
  1665. if port:
  1666. if port!=80 and port!=443:
  1667. if ':' not in domain:
  1668. domain=domain+':'+str(port)
  1669. if not pageNumber:
  1670. pageNumber=1
  1671. pageStr='?page='+str(pageNumber)
  1672. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1673. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1674. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1675. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1676. 'totalItems': 0,
  1677. 'type': 'OrderedCollection'}
  1678. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1679. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1680. 'orderedItems': [
  1681. ],
  1682. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1683. 'type': 'OrderedCollectionPage'}
  1684. if isModerator(baseDir,nickname):
  1685. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  1686. if os.path.isfile(moderationIndexFile):
  1687. with open(moderationIndexFile, "r") as f:
  1688. lines = f.readlines()
  1689. boxHeader['totalItems']=len(lines)
  1690. if headerOnly:
  1691. return boxHeader
  1692. pageLines=[]
  1693. if len(lines)>0:
  1694. endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
  1695. if endLineNumber<0:
  1696. endLineNumber=0
  1697. startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
  1698. if startLineNumber<0:
  1699. startLineNumber=0
  1700. lineNumber=startLineNumber
  1701. while lineNumber>=endLineNumber:
  1702. pageLines.append(lines[lineNumber].strip('\n'))
  1703. lineNumber-=1
  1704. for postUrl in pageLines:
  1705. postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
  1706. if os.path.isfile(postFilename):
  1707. postJsonObject=loadJson(postFilename)
  1708. if postJsonObject:
  1709. boxItems['orderedItems'].append(postJsonObject)
  1710. if headerOnly:
  1711. return boxHeader
  1712. return boxItems
  1713. def getStatusNumberFromPostFilename(filename) -> int:
  1714. """Gets the status number from a post filename
  1715. eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
  1716. returns 156295850695206
  1717. """
  1718. if '#statuses#' not in filename:
  1719. return None
  1720. return int(filename.split('#')[-1].replace('.json',''))
  1721. def isDM(postJsonObject: {}) -> bool:
  1722. """Returns true if the given post is a DM
  1723. """
  1724. if postJsonObject['type']!='Create':
  1725. return False
  1726. if not postJsonObject.get('object'):
  1727. return False
  1728. if not isinstance(postJsonObject['object'], dict):
  1729. return False
  1730. if postJsonObject['object']['type']!='Note':
  1731. return False
  1732. if postJsonObject['object'].get('moderationStatus'):
  1733. return False
  1734. fields=['to','cc']
  1735. for f in fields:
  1736. if not postJsonObject['object'].get(f):
  1737. continue
  1738. for toAddress in postJsonObject['object'][f]:
  1739. if toAddress.endswith('#Public'):
  1740. return False
  1741. if toAddress.endswith('followers'):
  1742. return False
  1743. return True
  1744. def isImageMedia(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {}) -> bool:
  1745. """Returns true if the given post has attached image media
  1746. """
  1747. if postJsonObject['type']=='Announce':
  1748. postJsonAnnounce=downloadAnnounce(session,baseDir,httpPrefix,nickname,domain,postJsonObject,__version__)
  1749. if postJsonAnnounce:
  1750. postJsonObject=postJsonAnnounce
  1751. if postJsonObject['type']!='Create':
  1752. return False
  1753. if not postJsonObject.get('object'):
  1754. return False
  1755. if not isinstance(postJsonObject['object'], dict):
  1756. return False
  1757. if postJsonObject['object'].get('moderationStatus'):
  1758. return False
  1759. if postJsonObject['object']['type']!='Note':
  1760. return False
  1761. if not postJsonObject['object'].get('attachment'):
  1762. return False
  1763. if not isinstance(postJsonObject['object']['attachment'], list):
  1764. return False
  1765. for attach in postJsonObject['object']['attachment']:
  1766. if attach.get('mediaType') and attach.get('url'):
  1767. if attach['mediaType'].startswith('image/') or \
  1768. attach['mediaType'].startswith('audio/') or \
  1769. attach['mediaType'].startswith('video/'):
  1770. return True
  1771. return False
  1772. def isReply(postJsonObject: {},actor: str) -> bool:
  1773. """Returns true if the given post is a reply to the given actor
  1774. """
  1775. if postJsonObject['type']!='Create':
  1776. return False
  1777. if not postJsonObject.get('object'):
  1778. return False
  1779. if not isinstance(postJsonObject['object'], dict):
  1780. return False
  1781. if postJsonObject['object'].get('moderationStatus'):
  1782. return False
  1783. if postJsonObject['object']['type']!='Note':
  1784. return False
  1785. if postJsonObject['object'].get('inReplyTo'):
  1786. if postJsonObject['object']['inReplyTo'].startswith(actor):
  1787. return True
  1788. if not postJsonObject['object'].get('tag'):
  1789. return False
  1790. if not isinstance(postJsonObject['object']['tag'], list):
  1791. return False
  1792. for tag in postJsonObject['object']['tag']:
  1793. if not tag.get('type'):
  1794. continue
  1795. if tag['type']=='Mention':
  1796. if not tag.get('href'):
  1797. continue
  1798. if actor in tag['href']:
  1799. return True
  1800. return False
  1801. def createBoxIndex(boxDir: str,postsInBoxDict: {}) -> int:
  1802. """ Creates an index for the given box
  1803. """
  1804. postsCtr=0
  1805. postsInPersonInbox=os.scandir(boxDir)
  1806. for postFilename in postsInPersonInbox:
  1807. postFilename=postFilename.name
  1808. if not postFilename.endswith('.json'):
  1809. continue
  1810. # extract the status number
  1811. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1812. if statusNumber:
  1813. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1814. postsCtr+=1
  1815. return postsCtr
  1816. def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
  1817. postsInBoxDict: {},postsCtr: int, \
  1818. nickname: str,domain: str, \
  1819. ocapAlways: bool) -> int:
  1820. """ Creates an index for the given shared inbox
  1821. """
  1822. handle=nickname+'@'+domain
  1823. followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
  1824. postsInSharedInbox=os.scandir(sharedBoxDir)
  1825. followingHandles=None
  1826. for postFilename in postsInSharedInbox:
  1827. postFilename=postFilename.name
  1828. if not postFilename.endswith('.json'):
  1829. continue
  1830. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1831. if not statusNumber:
  1832. continue
  1833. sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
  1834. # get the actor from the shared post
  1835. try:
  1836. with open(sharedInboxFilename, 'r') as fp:
  1837. postJsonObject=commentjson.load(fp)
  1838. except:
  1839. print('WARN: commentjson exception createSharedInboxIndex')
  1840. continue
  1841. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  1842. if not actorNickname:
  1843. continue
  1844. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  1845. if not actorDomain:
  1846. continue
  1847. # is the actor followed by this account?
  1848. if not followingHandles:
  1849. with open(followingFilename, 'r') as followingFile:
  1850. followingHandles = followingFile.read()
  1851. if actorNickname+'@'+actorDomain not in followingHandles:
  1852. continue
  1853. if ocapAlways:
  1854. capsList=None
  1855. # Note: should this be in the Create or the object of a post?
  1856. if postJsonObject.get('capability'):
  1857. if isinstance(postJsonObject['capability'], list):
  1858. capsList=postJsonObject['capability']
  1859. # Have capabilities been granted for the sender?
  1860. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
  1861. if not os.path.isfile(ocapFilename):
  1862. continue
  1863. # read the capabilities id
  1864. loadedOcap=False
  1865. try:
  1866. with open(ocapFilename, 'r') as fp:
  1867. ocapJson=commentjson.load(fp)
  1868. loadedOcap=True
  1869. except:
  1870. print('WARN: commentjson exception createSharedInboxIndex')
  1871. if loadedOcap:
  1872. if ocapJson.get('id'):
  1873. if ocapJson['id'] in capsList:
  1874. postsInBoxDict[statusNumber]=sharedInboxFilename
  1875. postsCtr+=1
  1876. else:
  1877. postsInBoxDict[statusNumber]=sharedInboxFilename
  1878. postsCtr+=1
  1879. return postsCtr
  1880. def addPostToTimeline(filePath: str,boxname: str,postsInBox: [],boxActor: str) -> bool:
  1881. """ is this a valid timeline post?
  1882. """
  1883. # must be a "Note" or "Announce" type
  1884. with open(filePath, 'r') as postFile:
  1885. postStr = postFile.read()
  1886. if '"Note"' in postStr or '"Announce"' in postStr or \
  1887. ('"Question"' in postStr and '"Create"' in postStr):
  1888. if boxname=='dm':
  1889. if '#Public' in postStr or '/followers' in postStr:
  1890. return False
  1891. elif boxname=='tlreplies':
  1892. if boxActor not in postStr:
  1893. return False
  1894. elif boxname=='tlmedia':
  1895. if '"Create"' in postStr:
  1896. if 'mediaType' not in postStr or 'image/' not in postStr:
  1897. return False
  1898. # add the post to the dictionary
  1899. postsInBox.append(postStr)
  1900. return True
  1901. return False
  1902. def createBoxIndexed(session,baseDir: str,boxname: str, \
  1903. nickname: str,domain: str,port: int,httpPrefix: str, \
  1904. itemsPerPage: int,headerOnly: bool,authorized :bool, \
  1905. ocapAlways: bool,pageNumber=None) -> {}:
  1906. """Constructs the box feed for a person with the given nickname
  1907. """
  1908. if not authorized or not pageNumber:
  1909. pageNumber=1
  1910. if boxname!='inbox' and boxname!='dm' and \
  1911. boxname!='tlreplies' and boxname!='tlmedia' and \
  1912. boxname!='outbox' and boxname!='tlbookmarks':
  1913. return None
  1914. if boxname!='dm' and boxname!='tlreplies' and \
  1915. boxname!='tlmedia' and boxname!='tlbookmarks':
  1916. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1917. else:
  1918. # extract DMs or replies or media from the inbox
  1919. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1920. announceCacheDir=baseDir+'/cache/announce/'+nickname
  1921. sharedBoxDir=None
  1922. if boxname=='inbox' or boxname=='tlreplies' or \
  1923. boxname=='tlmedia':
  1924. sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
  1925. # bookmarks timeline is like the inbox but has its own separate index
  1926. indexBoxName=boxname
  1927. if boxname=='tlbookmarks':
  1928. indexBoxName='bookmarks'
  1929. elif boxname=='dm':
  1930. indexBoxName='dm'
  1931. elif boxname=='tlreplies':
  1932. indexBoxName='tlreplies'
  1933. elif boxname=='tlmedia':
  1934. indexBoxName='tlmedia'
  1935. if port:
  1936. if port!=80 and port!=443:
  1937. if ':' not in domain:
  1938. domain=domain+':'+str(port)
  1939. boxActor=httpPrefix+'://'+domain+'/users/'+nickname
  1940. pageStr='?page=true'
  1941. if pageNumber:
  1942. try:
  1943. pageStr='?page='+str(pageNumber)
  1944. except:
  1945. pass
  1946. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1947. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1948. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1949. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1950. 'totalItems': 0,
  1951. 'type': 'OrderedCollection'}
  1952. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1953. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1954. 'orderedItems': [
  1955. ],
  1956. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1957. 'type': 'OrderedCollectionPage'}
  1958. postsInBox=[]
  1959. indexFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+indexBoxName+'.index'
  1960. postsCtr=0
  1961. if os.path.isfile(indexFilename):
  1962. maxPostCtr=itemsPerPage*pageNumber
  1963. with open(indexFilename, 'r') as indexFile:
  1964. while postsCtr<maxPostCtr:
  1965. postFilename=indexFile.readline()
  1966. if not postFilename:
  1967. postsCtr+=1
  1968. continue
  1969. # Skip through any posts previous to the current page
  1970. if postsCtr<int((pageNumber-1)*itemsPerPage):
  1971. postsCtr+=1
  1972. continue
  1973. # if this is a full path then remove the directories
  1974. if '/' in postFilename:
  1975. postFilename=postFilename.split('/')[-1]
  1976. # filename of the post without any extension or path
  1977. postUrl=postFilename.replace('\n','').replace('.json','').strip()
  1978. # get the full path of the post
  1979. fullPostFilename= \
  1980. locatePost(baseDir,nickname,domain,postUrl,False)
  1981. if fullPostFilename:
  1982. addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
  1983. else:
  1984. print('WARN: unable to locate post '+postUrl)
  1985. postsCtr+=1
  1986. # Generate first and last entries within header
  1987. if postsCtr>0:
  1988. lastPage=int(postsCtr/itemsPerPage)
  1989. if lastPage<1:
  1990. lastPage=1
  1991. boxHeader['last']= \
  1992. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
  1993. if headerOnly:
  1994. boxHeader['totalItems']=len(postsInBox)
  1995. prevPageStr='true'
  1996. if pageNumber>1:
  1997. prevPageStr=str(pageNumber-1)
  1998. boxHeader['prev']= \
  1999. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+prevPageStr
  2000. nextPageStr=str(pageNumber+1)
  2001. boxHeader['next']= \
  2002. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+nextPageStr
  2003. return boxHeader
  2004. for postStr in postsInBox:
  2005. p=None
  2006. try:
  2007. p=json.loads(postStr)
  2008. except:
  2009. continue
  2010. # remove any capability so that it's not displayed
  2011. if p.get('capability'):
  2012. del p['capability']
  2013. # Don't show likes, replies or shares (announces) to unauthorized viewers
  2014. if not authorized:
  2015. if p.get('object'):
  2016. if isinstance(p['object'], dict):
  2017. if p['object'].get('likes'):
  2018. p['likes']={}
  2019. if p['object'].get('replies'):
  2020. p['replies']={}
  2021. if p['object'].get('shares'):
  2022. p['shares']={}
  2023. if p['object'].get('bookmarks'):
  2024. p['bookmarks']={}
  2025. boxItems['orderedItems'].append(p)
  2026. return boxItems
  2027. def expireCache(baseDir: str,personCache: {},httpPrefix: str,archiveDir: str,maxPostsInBox=32000):
  2028. """Thread used to expire actors from the cache and archive old posts
  2029. """
  2030. while True:
  2031. # once per day
  2032. time.sleep(60*60*24)
  2033. expirePersonCache(basedir,personCache)
  2034. archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
  2035. def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str,maxPostsInBox=32000) -> None:
  2036. """Archives posts for all accounts
  2037. """
  2038. if archiveDir:
  2039. if not os.path.isdir(archiveDir):
  2040. os.mkdir(archiveDir)
  2041. if archiveDir:
  2042. if not os.path.isdir(archiveDir+'/accounts'):
  2043. os.mkdir(archiveDir+'/accounts')
  2044. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  2045. for handle in dirs:
  2046. if '@' in handle:
  2047. nickname=handle.split('@')[0]
  2048. domain=handle.split('@')[1]
  2049. archiveSubdir=None
  2050. if archiveDir:
  2051. if not os.path.isdir(archiveDir+'/accounts/'+handle):
  2052. os.mkdir(archiveDir+'/accounts/'+handle)
  2053. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
  2054. os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
  2055. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
  2056. os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
  2057. archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
  2058. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2059. 'inbox',archiveSubdir, \
  2060. maxPostsInBox)
  2061. if archiveDir:
  2062. archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
  2063. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2064. 'outbox',archiveSubdir, \
  2065. maxPostsInBox)
  2066. def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
  2067. boxname: str,archiveDir: str,maxPostsInBox=32000) -> None:
  2068. """Retain a maximum number of posts within the given box
  2069. Move any others to an archive directory
  2070. """
  2071. if boxname!='inbox' and boxname!='outbox':
  2072. return
  2073. if archiveDir:
  2074. if not os.path.isdir(archiveDir):
  2075. os.mkdir(archiveDir)
  2076. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  2077. postsInBox=os.scandir(boxDir)
  2078. noOfPosts=0
  2079. for f in postsInBox:
  2080. noOfPosts+=1
  2081. if noOfPosts<=maxPostsInBox:
  2082. return
  2083. # remove entries from the index
  2084. handle=nickname+'@'+domain
  2085. indexFilename=baseDir+'/accounts/'+handle+'/'+boxname+'.index'
  2086. if os.path.isfile(indexFilename):
  2087. indexCtr=0
  2088. # get the existing index entries as a string
  2089. newIndex=''
  2090. with open(indexFilename, 'r') as indexFile:
  2091. for postId in indexFile:
  2092. newIndex+=postId
  2093. indexCtr+=1
  2094. if indexCtr>=maxPostsInBox:
  2095. break
  2096. # save the new index file
  2097. if len(newIndex)>0:
  2098. indexFile=open(indexFilename,'w+')
  2099. if indexFile:
  2100. indexFile.write(newIndex)
  2101. indexFile.close()
  2102. postsInBoxDict={}
  2103. postsCtr=0
  2104. postsInBox=os.scandir(boxDir)
  2105. for postFilename in postsInBox:
  2106. postFilename=postFilename.name
  2107. if not postFilename.endswith('.json'):
  2108. continue
  2109. # Time of file creation
  2110. fullFilename=os.path.join(boxDir,postFilename)
  2111. if os.path.isfile(fullFilename):
  2112. content=open(fullFilename).read()
  2113. if '"published":' in content:
  2114. publishedStr=content.split('"published":')[1]
  2115. if '"' in publishedStr:
  2116. publishedStr=publishedStr.split('"')[1]
  2117. if publishedStr.endswith('Z'):
  2118. postsInBoxDict[publishedStr]=postFilename
  2119. postsCtr+=1
  2120. noOfPosts=postsCtr
  2121. if noOfPosts<=maxPostsInBox:
  2122. return
  2123. # sort the list in ascending order of date
  2124. postsInBoxSorted=OrderedDict(sorted(postsInBoxDict.items(),reverse=False))
  2125. # directory containing cached html posts
  2126. postCacheDir=boxDir.replace('/'+boxname,'/postcache')
  2127. for publishedStr,postFilename in postsInBoxSorted.items():
  2128. filePath=os.path.join(boxDir,postFilename)
  2129. if not os.path.isfile(filePath):
  2130. continue
  2131. if archiveDir:
  2132. repliesPath=filePath.replace('.json','.replies')
  2133. archivePath=os.path.join(archiveDir,postFilename)
  2134. os.rename(filePath,archivePath)
  2135. if os.path.isfile(repliesPath):
  2136. os.rename(repliesPath,archivePath)
  2137. else:
  2138. deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
  2139. # remove cached html posts
  2140. postCacheFilename=os.path.join(postCacheDir,postFilename).replace('.json','.html')
  2141. if os.path.isfile(postCacheFilename):
  2142. os.remove(postCacheFilename)
  2143. noOfPosts-=1
  2144. if noOfPosts<=maxPostsInBox:
  2145. break
  2146. def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
  2147. raw: bool,simple: bool,useTor: bool, \
  2148. port: int,httpPrefix: str, \
  2149. debug: bool,projectVersion: str) -> None:
  2150. """ This is really just for test purposes
  2151. """
  2152. session = createSession(useTor)
  2153. personCache={}
  2154. cachedWebfingers={}
  2155. federationList=[]
  2156. domainFull=domain
  2157. if port:
  2158. if port!=80 and port!=443:
  2159. if ':' not in domain:
  2160. domainFull=domain+':'+str(port)
  2161. handle=httpPrefix+"://"+domainFull+"/@"+nickname
  2162. wfRequest = \
  2163. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  2164. domain,projectVersion)
  2165. if not wfRequest:
  2166. sys.exit()
  2167. personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
  2168. getPersonBox(baseDir,session,wfRequest,personCache, \
  2169. projectVersion,httpPrefix,nickname,domain,'outbox')
  2170. wfResult = json.dumps(wfRequest, indent=2, sort_keys=False)
  2171. maxMentions=10
  2172. maxEmoji=10
  2173. maxAttachments=5
  2174. userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
  2175. maxAttachments,federationList, \
  2176. personCache,raw,simple,debug, \
  2177. projectVersion,httpPrefix,domain)
  2178. #print(str(userPosts))
  2179. def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
  2180. nickname: str,domain: str,port: int, \
  2181. followerUrl,updateCaps: [], \
  2182. sendThreads: [],postLog: [], \
  2183. cachedWebfingers: {},personCache: {}, \
  2184. federationList :[],debug :bool, \
  2185. projectVersion: str) -> int:
  2186. """When the capabilities for a follower are changed this
  2187. sends out an update. followerUrl is the actor of the follower.
  2188. """
  2189. updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
  2190. nickname,domain,port, \
  2191. followerUrl, \
  2192. updateCaps)
  2193. if not updateJson:
  2194. return 1
  2195. if debug:
  2196. pprint(updateJson)
  2197. print('DEBUG: sending capabilities update from '+ \
  2198. nickname+'@'+domain+' port '+ str(port) + \
  2199. ' to '+followerUrl)
  2200. clientToServer=False
  2201. followerNickname=getNicknameFromActor(followerUrl)
  2202. if not followerNickname:
  2203. print('WARN: unable to find nickname in '+followerUrl)
  2204. return 1
  2205. followerDomain,followerPort=getDomainFromActor(followerUrl)
  2206. return sendSignedJson(updateJson,session,baseDir, \
  2207. nickname,domain,port, \
  2208. followerNickname,followerDomain,followerPort, '', \
  2209. httpPrefix,True,clientToServer, \
  2210. federationList, \
  2211. sendThreads,postLog,cachedWebfingers, \
  2212. personCache,debug,projectVersion)
  2213. def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
  2214. postRepliesFilename: str,authorized: bool, \
  2215. repliesJson: {}) -> None:
  2216. # populate the items list with replies
  2217. repliesBoxes=['outbox','inbox']
  2218. with open(postRepliesFilename,'r') as repliesFile:
  2219. for messageId in repliesFile:
  2220. replyFound=False
  2221. # examine inbox and outbox
  2222. for boxname in repliesBoxes:
  2223. searchFilename= \
  2224. baseDir+ \
  2225. '/accounts/'+nickname+'@'+ \
  2226. domain+'/'+ \
  2227. boxname+'/'+ \
  2228. messageId.replace('\n','').replace('/','#')+'.json'
  2229. if os.path.isfile(searchFilename):
  2230. if authorized or \
  2231. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2232. postJsonObject=loadJson(searchFilename)
  2233. if postJsonObject:
  2234. if postJsonObject['object'].get('cc'):
  2235. if authorized or \
  2236. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2237. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2238. repliesJson['orderedItems'].append(postJsonObject)
  2239. replyFound=True
  2240. else:
  2241. if authorized or \
  2242. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2243. repliesJson['orderedItems'].append(postJsonObject)
  2244. replyFound=True
  2245. break
  2246. # if not in either inbox or outbox then examine the shared inbox
  2247. if not replyFound:
  2248. searchFilename= \
  2249. baseDir+ \
  2250. '/accounts/inbox@'+ \
  2251. domain+'/inbox/'+ \
  2252. messageId.replace('\n','').replace('/','#')+'.json'
  2253. if os.path.isfile(searchFilename):
  2254. if authorized or \
  2255. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2256. # get the json of the reply and append it to the collection
  2257. postJsonObject=loadJson(searchFilename)
  2258. if postJsonObject:
  2259. if postJsonObject['object'].get('cc'):
  2260. if authorized or \
  2261. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2262. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2263. repliesJson['orderedItems'].append(postJsonObject)
  2264. else:
  2265. if authorized or \
  2266. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2267. repliesJson['orderedItems'].append(postJsonObject)
  2268. def rejectAnnounce(announceFilename: str):
  2269. """Marks an announce as rejected
  2270. """
  2271. if not os.path.isfile(announceFilename+'.reject'):
  2272. rejectAnnounceFile=open(announceFilename+'.reject', "w+")
  2273. rejectAnnounceFile.write('\n')
  2274. rejectAnnounceFile.close()
  2275. def downloadAnnounce(session,baseDir: str,httpPrefix: str,nickname: str,domain: str,postJsonObject: {},projectVersion: str) -> {}:
  2276. """Download the post referenced by an announce
  2277. """
  2278. if not postJsonObject.get('object'):
  2279. return None
  2280. if not isinstance(postJsonObject['object'], str):
  2281. return None
  2282. # get the announced post
  2283. announceCacheDir=baseDir+'/cache/announce/'+nickname
  2284. if not os.path.isdir(announceCacheDir):
  2285. os.mkdir(announceCacheDir)
  2286. announceFilename=announceCacheDir+'/'+postJsonObject['object'].replace('/','#')+'.json'
  2287. print('announceFilename: '+announceFilename)
  2288. if os.path.isfile(announceFilename+'.reject'):
  2289. return None
  2290. if os.path.isfile(announceFilename):
  2291. print('Reading cached Announce content for '+postJsonObject['object'])
  2292. postJsonObject=loadJson(announceFilename)
  2293. if postJsonObject:
  2294. return postJsonObject
  2295. else:
  2296. print('Downloading Announce content for '+postJsonObject['object'])
  2297. asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  2298. if '/channel/' in postJsonObject['actor']:
  2299. asHeader={'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  2300. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  2301. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  2302. announcedJson=getJson(session,postJsonObject['object'],asHeader,None,projectVersion,httpPrefix,domain)
  2303. if not announcedJson:
  2304. return None
  2305. if not announcedJson.get('id'):
  2306. rejectAnnounce(announceFilename)
  2307. #pprint(announcedJson)
  2308. return None
  2309. if '/statuses/' not in announcedJson['id']:
  2310. rejectAnnounce(announceFilename)
  2311. return None
  2312. if '/users/' not in announcedJson['id'] and \
  2313. '/channel/' not in announcedJson['id'] and \
  2314. '/profile/' not in announcedJson['id']:
  2315. rejectAnnounce(announceFilename)
  2316. return None
  2317. if not announcedJson.get('type'):
  2318. rejectAnnounce(announceFilename)
  2319. #pprint(announcedJson)
  2320. return None
  2321. if announcedJson['type']!='Note':
  2322. rejectAnnounce(announceFilename)
  2323. #pprint(announcedJson)
  2324. return None
  2325. # wrap in create to be consistent with other posts
  2326. announcedJson= \
  2327. outboxMessageCreateWrap(httpPrefix, \
  2328. actorNickname,actorDomain,actorPort, \
  2329. announcedJson)
  2330. if announcedJson['type']!='Create':
  2331. rejectAnnounce(announceFilename)
  2332. #pprint(announcedJson)
  2333. return None
  2334. # set the id to the original status
  2335. announcedJson['id']=postJsonObject['object']
  2336. announcedJson['object']['id']=postJsonObject['object']
  2337. # check that the repeat isn't for a blocked account
  2338. attributedNickname=getNicknameFromActor(announcedJson['object']['id'])
  2339. attributedDomain,attributedPort=getDomainFromActor(announcedJson['object']['id'])
  2340. if attributedNickname and attributedDomain:
  2341. if attributedPort:
  2342. if attributedPort!=80 and attributedPort!=443:
  2343. attributedDomain=attributedDomain+':'+str(attributedPort)
  2344. if isBlocked(baseDir,nickname,domain,attributedNickname,attributedDomain):
  2345. rejectAnnounce(announceFilename)
  2346. return None
  2347. postJsonObject=announcedJson
  2348. if saveJson(postJsonObject,announceFilename):
  2349. return postJsonObject
  2350. return None