posts.py 83 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import requests
  9. import json
  10. import commentjson
  11. import html
  12. import datetime
  13. import os
  14. import shutil
  15. import threading
  16. import sys
  17. import trace
  18. import time
  19. from collections import OrderedDict
  20. from threads import threadWithTrace
  21. from cache import storePersonInCache
  22. from cache import getPersonFromCache
  23. from cache import expirePersonCache
  24. from pprint import pprint
  25. from random import randint
  26. from session import createSession
  27. from session import getJson
  28. from session import postJsonString
  29. from session import postImage
  30. from webfinger import webfingerHandle
  31. from httpsig import createSignedHeader
  32. from utils import getStatusNumber
  33. from utils import createPersonDir
  34. from utils import urlPermitted
  35. from utils import getNicknameFromActor
  36. from utils import getDomainFromActor
  37. from utils import deletePost
  38. from utils import validNickname
  39. from capabilities import getOcapFilename
  40. from capabilities import capabilitiesUpdate
  41. from media import attachMedia
  42. from content import addHtmlTags
  43. from auth import createBasicAuthHeader
  44. from config import getConfigParam
  45. try:
  46. from BeautifulSoup import BeautifulSoup
  47. except ImportError:
  48. from bs4 import BeautifulSoup
  49. def isModerator(baseDir: str,nickname: str) -> bool:
  50. """Returns true if the given nickname is a moderator
  51. """
  52. moderatorsFile=baseDir+'/accounts/moderators.txt'
  53. if not os.path.isfile(moderatorsFile):
  54. if getConfigParam(baseDir,'admin')==nickname:
  55. return True
  56. return False
  57. with open(moderatorsFile, "r") as f:
  58. lines = f.readlines()
  59. if len(lines)==0:
  60. if getConfigParam(baseDir,'admin')==nickname:
  61. return True
  62. for moderator in lines:
  63. moderator=moderator.strip('\n')
  64. if moderator==nickname:
  65. return True
  66. return False
  67. def noOfFollowersOnDomain(baseDir: str,handle: str, \
  68. domain: str, followFile='followers.txt') -> int:
  69. """Returns the number of followers of the given handle from the given domain
  70. """
  71. filename=baseDir+'/accounts/'+handle+'/'+followFile
  72. if not os.path.isfile(filename):
  73. return 0
  74. ctr=0
  75. with open(filename, "r") as followersFilename:
  76. for followerHandle in followersFilename:
  77. if '@' in followerHandle:
  78. followerDomain= \
  79. followerHandle.split('@')[1].replace('\n','')
  80. if domain==followerDomain:
  81. ctr+=1
  82. return ctr
  83. def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
  84. debug=False):
  85. """Returns the public or private key of a person
  86. """
  87. handle=nickname+'@'+domain
  88. keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
  89. if not os.path.isfile(keyFilename):
  90. if debug:
  91. print('DEBUG: private key file not found: '+keyFilename)
  92. return ''
  93. keyPem=''
  94. with open(keyFilename, "r") as pemFile:
  95. keyPem=pemFile.read()
  96. if len(keyPem)<20:
  97. if debug:
  98. print('DEBUG: private key was too short: '+keyPem)
  99. return ''
  100. return keyPem
  101. def cleanHtml(rawHtml: str) -> str:
  102. text = BeautifulSoup(rawHtml, 'html.parser').get_text()
  103. return html.unescape(text)
  104. def getUserUrl(wfRequest) -> str:
  105. if wfRequest.get('links'):
  106. for link in wfRequest['links']:
  107. if link.get('type') and link.get('href'):
  108. if link['type'] == 'application/activity+json':
  109. if '/users/' in link['href'] or '/profile/' in link['href']:
  110. return link['href']
  111. else:
  112. print('Webfinger activity+json does not contain a valid actor')
  113. print(link['href'])
  114. return None
  115. def parseUserFeed(session,feedUrl: str,asHeader: {}, \
  116. projectVersion: str,httpPrefix: str,domain: str) -> None:
  117. feedJson = getJson(session,feedUrl,asHeader,None, \
  118. projectVersion,httpPrefix,domain)
  119. if not feedJson:
  120. return
  121. if 'orderedItems' in feedJson:
  122. for item in feedJson['orderedItems']:
  123. yield item
  124. nextUrl = None
  125. if 'first' in feedJson:
  126. nextUrl = feedJson['first']
  127. elif 'next' in feedJson:
  128. nextUrl = feedJson['next']
  129. if nextUrl:
  130. if isinstance(nextUrl, str):
  131. userFeed=parseUserFeed(session,nextUrl,asHeader, \
  132. projectVersion,httpPrefix,domain)
  133. for item in userFeed:
  134. yield item
  135. elif isinstance(nextUrl, dict):
  136. userFeed=nextUrl
  137. if userFeed.get('orderedItems'):
  138. for item in userFeed['orderedItems']:
  139. yield item
  140. def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
  141. projectVersion: str,httpPrefix: str,domain: str, \
  142. boxName='inbox') -> (str,str,str,str,str,str,str,str):
  143. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  144. personUrl = getUserUrl(wfRequest)
  145. if not personUrl:
  146. return None,None,None,None,None,None,None,None
  147. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  148. if not personJson:
  149. personJson = getJson(session,personUrl,asHeader,None, \
  150. projectVersion,httpPrefix,domain)
  151. if not personJson:
  152. return None,None,None,None,None,None,None,None
  153. boxJson=None
  154. if not personJson.get(boxName):
  155. if personJson.get('endpoints'):
  156. if personJson['endpoints'].get(boxName):
  157. boxJson=personJson['endpoints'][boxName]
  158. else:
  159. boxJson=personJson[boxName]
  160. if not boxJson:
  161. return None,None,None,None,None,None,None,None
  162. personId=None
  163. if personJson.get('id'):
  164. personId=personJson['id']
  165. pubKeyId=None
  166. pubKey=None
  167. if personJson.get('publicKey'):
  168. if personJson['publicKey'].get('id'):
  169. pubKeyId=personJson['publicKey']['id']
  170. if personJson['publicKey'].get('publicKeyPem'):
  171. pubKey=personJson['publicKey']['publicKeyPem']
  172. sharedInbox=None
  173. if personJson.get('sharedInbox'):
  174. sharedInbox=personJson['sharedInbox']
  175. else:
  176. if personJson.get('endpoints'):
  177. if personJson['endpoints'].get('sharedInbox'):
  178. sharedInbox=personJson['endpoints']['sharedInbox']
  179. capabilityAcquisition=None
  180. if personJson.get('capabilityAcquisitionEndpoint'):
  181. capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
  182. avatarUrl=None
  183. if personJson.get('icon'):
  184. if personJson['icon'].get('url'):
  185. avatarUrl=personJson['icon']['url']
  186. displayName=None
  187. if personJson.get('name'):
  188. displayName=personJson['name']
  189. storePersonInCache(baseDir,personUrl,personJson,personCache)
  190. return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
  191. def getPosts(session,outboxUrl: str,maxPosts: int, \
  192. maxMentions: int, \
  193. maxEmoji: int,maxAttachments: int, \
  194. federationList: [], \
  195. personCache: {},raw: bool, \
  196. simple: bool,debug: bool, \
  197. projectVersion: str,httpPrefix: str,domain: str) -> {}:
  198. """Gets public posts from an outbox
  199. """
  200. personPosts={}
  201. if not outboxUrl:
  202. return personPosts
  203. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  204. if raw:
  205. result = []
  206. i = 0
  207. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  208. projectVersion,httpPrefix,domain)
  209. for item in userFeed:
  210. result.append(item)
  211. i += 1
  212. if i == maxPosts:
  213. break
  214. pprint(result)
  215. return None
  216. i = 0
  217. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  218. projectVersion,httpPrefix,domain)
  219. for item in userFeed:
  220. if not item.get('id'):
  221. if debug:
  222. print('No id')
  223. continue
  224. if not item.get('type'):
  225. if debug:
  226. print('No type')
  227. continue
  228. if item['type'] != 'Create':
  229. if debug:
  230. print('Not Create type')
  231. continue
  232. if not item.get('object'):
  233. if debug:
  234. print('No object')
  235. continue
  236. if not isinstance(item['object'], dict):
  237. if debug:
  238. print('item object is not a dict')
  239. continue
  240. if not item['object'].get('published'):
  241. if debug:
  242. print('No published attribute')
  243. continue
  244. #pprint(item)
  245. published = item['object']['published']
  246. if not personPosts.get(item['id']):
  247. # check that this is a public post
  248. # #Public should appear in the "to" list
  249. if item['object'].get('to'):
  250. isPublic=False
  251. for recipient in item['object']['to']:
  252. if recipient.endswith('#Public'):
  253. isPublic=True
  254. break
  255. if not isPublic:
  256. continue
  257. content = item['object']['content'].replace('&apos;',"'")
  258. mentions=[]
  259. emoji={}
  260. if item['object'].get('tag'):
  261. for tagItem in item['object']['tag']:
  262. tagType=tagItem['type'].lower()
  263. if tagType=='emoji':
  264. if tagItem.get('name') and tagItem.get('icon'):
  265. if tagItem['icon'].get('url'):
  266. # No emoji from non-permitted domains
  267. if urlPermitted(tagItem['icon']['url'], \
  268. federationList, \
  269. "objects:read"):
  270. emojiName=tagItem['name']
  271. emojiIcon=tagItem['icon']['url']
  272. emoji[emojiName]=emojiIcon
  273. else:
  274. if debug:
  275. print('url not permitted '+tagItem['icon']['url'])
  276. if tagType=='mention':
  277. if tagItem.get('name'):
  278. if tagItem['name'] not in mentions:
  279. mentions.append(tagItem['name'])
  280. if len(mentions)>maxMentions:
  281. if debug:
  282. print('max mentions reached')
  283. continue
  284. if len(emoji)>maxEmoji:
  285. if debug:
  286. print('max emojis reached')
  287. continue
  288. summary = ''
  289. if item['object'].get('summary'):
  290. if item['object']['summary']:
  291. summary = item['object']['summary']
  292. inReplyTo = ''
  293. if item['object'].get('inReplyTo'):
  294. if item['object']['inReplyTo']:
  295. # No replies to non-permitted domains
  296. if not urlPermitted(item['object']['inReplyTo'], \
  297. federationList, \
  298. "objects:read"):
  299. if debug:
  300. print('url not permitted '+item['object']['inReplyTo'])
  301. continue
  302. inReplyTo = item['object']['inReplyTo']
  303. conversation = ''
  304. if item['object'].get('conversation'):
  305. if item['object']['conversation']:
  306. # no conversations originated in non-permitted domains
  307. if urlPermitted(item['object']['conversation'], \
  308. federationList,"objects:read"):
  309. conversation = item['object']['conversation']
  310. attachment = []
  311. if item['object'].get('attachment'):
  312. if item['object']['attachment']:
  313. for attach in item['object']['attachment']:
  314. if attach.get('name') and attach.get('url'):
  315. # no attachments from non-permitted domains
  316. if urlPermitted(attach['url'], \
  317. federationList, \
  318. "objects:read"):
  319. attachment.append([attach['name'],attach['url']])
  320. else:
  321. if debug:
  322. print('url not permitted '+attach['url'])
  323. sensitive = False
  324. if item['object'].get('sensitive'):
  325. sensitive = item['object']['sensitive']
  326. if simple:
  327. print(cleanHtml(content)+'\n')
  328. else:
  329. pprint(item)
  330. personPosts[item['id']] = {
  331. "sensitive": sensitive,
  332. "inreplyto": inReplyTo,
  333. "summary": summary,
  334. "html": content,
  335. "plaintext": cleanHtml(content),
  336. "attachment": attachment,
  337. "mentions": mentions,
  338. "emoji": emoji,
  339. "conversation": conversation
  340. }
  341. i += 1
  342. if i == maxPosts:
  343. break
  344. return personPosts
  345. def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
  346. """Deletes all posts for a person from inbox or outbox
  347. """
  348. if boxname!='inbox' and boxname!='outbox':
  349. return
  350. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  351. for deleteFilename in os.listdir(boxDir):
  352. filePath = os.path.join(boxDir, deleteFilename)
  353. try:
  354. if os.path.isfile(filePath):
  355. os.unlink(filePath)
  356. elif os.path.isdir(filePath): shutil.rmtree(filePath)
  357. except Exception as e:
  358. print(e)
  359. def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
  360. nickname: str, domain: str,postJsonObject: {}, \
  361. boxname: str) -> str:
  362. """Saves the give json to the give box
  363. Returns the filename
  364. """
  365. if boxname!='inbox' and boxname!='outbox':
  366. return None
  367. originalDomain=domain
  368. if ':' in domain:
  369. domain=domain.split(':')[0]
  370. if not postId:
  371. statusNumber,published = getStatusNumber()
  372. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  373. postJsonObject['id']=postId+'/activity'
  374. if postJsonObject.get('object'):
  375. if isinstance(postJsonObject['object'], dict):
  376. postJsonObject['object']['id']=postId
  377. postJsonObject['object']['atomUri']=postId
  378. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  379. filename=boxDir+'/'+postId.replace('/','#')+'.json'
  380. with open(filename, 'w') as fp:
  381. commentjson.dump(postJsonObject, fp, indent=4, sort_keys=False)
  382. return filename
  383. def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
  384. """Writes the post url for hashtags to a file
  385. This allows posts for a hashtag to be quickly looked up
  386. """
  387. if tag['type']!='Hashtag':
  388. return
  389. # create hashtags directory
  390. tagsDir=baseDir+'/tags'
  391. if not os.path.isdir(tagsDir):
  392. os.mkdir(tagsDir)
  393. tagName=tag['name']
  394. tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
  395. tagFile=open(tagsFilename, "a+")
  396. if not tagFile:
  397. return
  398. tagFile.write(newPostId+'\n')
  399. tagFile.close()
  400. def createPostBase(baseDir: str,nickname: str, domain: str, port: int, \
  401. toUrl: str, ccUrl: str, httpPrefix: str, content: str, \
  402. followersOnly: bool, saveToFile: bool, clientToServer: bool, \
  403. attachImageFilename: str,mediaType: str,imageDescription: str, \
  404. useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
  405. inReplyToAtomUri=None, subject=None) -> {}:
  406. """Creates a message
  407. """
  408. mentionedRecipients= \
  409. getMentionedPeople(baseDir,httpPrefix,content,domain,False)
  410. tags=[]
  411. hashtagsDict={}
  412. if port:
  413. if port!=80 and port!=443:
  414. if ':' not in domain:
  415. domain=domain+':'+str(port)
  416. # convert content to html
  417. content= \
  418. addHtmlTags(baseDir,httpPrefix, \
  419. nickname,domain,content, \
  420. mentionedRecipients, \
  421. hashtagsDict)
  422. statusNumber,published = getStatusNumber()
  423. postTo='https://www.w3.org/ns/activitystreams#Public'
  424. postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  425. if followersOnly:
  426. postTo=postCC
  427. postCC=''
  428. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  429. sensitive=False
  430. summary=None
  431. if subject:
  432. summary=subject
  433. sensitive=True
  434. toRecipients=[]
  435. if toUrl:
  436. if not isinstance(toUrl, str):
  437. print('ERROR: toUrl is not a string')
  438. return None
  439. toRecipients=[toUrl]
  440. # who to send to
  441. if mentionedRecipients:
  442. for mention in mentionedRecipients:
  443. if mention not in toRecipients:
  444. toRecipients.append(mention)
  445. # create a list of hashtags
  446. # Only posts which are #Public are searchable by hashtag
  447. if hashtagsDict:
  448. isPublic=False
  449. for recipient in toRecipients:
  450. if recipient.endswith('#Public'):
  451. isPublic=True
  452. break
  453. for tagName,tag in hashtagsDict.items():
  454. tags.append(tag)
  455. if isPublic:
  456. updateHashtagsIndex(baseDir,tag,newPostId)
  457. if not clientToServer:
  458. actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
  459. # if capabilities have been granted for this actor
  460. # then get the corresponding id
  461. capabilityId=None
  462. capabilityIdList=[]
  463. ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
  464. if ocapFilename:
  465. if os.path.isfile(ocapFilename):
  466. with open(ocapFilename, 'r') as fp:
  467. oc=commentjson.load(fp)
  468. if oc.get('id'):
  469. capabilityIdList=[oc['id']]
  470. newPost = {
  471. "@context": "https://www.w3.org/ns/activitystreams",
  472. 'id': newPostId+'/activity',
  473. 'capability': capabilityIdList,
  474. 'type': 'Create',
  475. 'actor': actorUrl,
  476. 'published': published,
  477. 'to': [toUrl],
  478. 'cc': [],
  479. 'object': {
  480. 'id': newPostId,
  481. 'type': 'Note',
  482. 'summary': summary,
  483. 'inReplyTo': inReplyTo,
  484. 'published': published,
  485. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  486. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  487. 'to': toRecipients,
  488. 'cc': [],
  489. 'sensitive': sensitive,
  490. 'atomUri': newPostId,
  491. 'inReplyToAtomUri': inReplyToAtomUri,
  492. 'content': content,
  493. 'contentMap': {
  494. 'en': content
  495. },
  496. 'attachment': [],
  497. 'tag': tags,
  498. 'replies': {
  499. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  500. 'type': 'Collection',
  501. 'first': {
  502. 'type': 'CollectionPage',
  503. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  504. 'items': []
  505. }
  506. }
  507. }
  508. }
  509. if attachImageFilename:
  510. newPost['object']= \
  511. attachMedia(baseDir,httpPrefix,domain,port, \
  512. newPost['object'],attachImageFilename, \
  513. mediaType,imageDescription,useBlurhash)
  514. else:
  515. newPost = {
  516. "@context": "https://www.w3.org/ns/activitystreams",
  517. 'id': newPostId,
  518. 'type': 'Note',
  519. 'summary': summary,
  520. 'inReplyTo': inReplyTo,
  521. 'published': published,
  522. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  523. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  524. 'to': toRecipients,
  525. 'cc': [],
  526. 'sensitive': sensitive,
  527. 'atomUri': newPostId,
  528. 'inReplyToAtomUri': inReplyToAtomUri,
  529. 'content': content,
  530. 'contentMap': {
  531. 'en': content
  532. },
  533. 'attachment': [],
  534. 'tag': tags,
  535. 'replies': {
  536. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  537. 'type': 'Collection',
  538. 'first': {
  539. 'type': 'CollectionPage',
  540. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  541. 'items': []
  542. }
  543. }
  544. }
  545. if attachImageFilename:
  546. newPost= \
  547. attachMedia(baseDir,httpPrefix,domain,port, \
  548. newPost,attachImageFilename, \
  549. mediaType,imageDescription,useBlurhash)
  550. if ccUrl:
  551. if len(ccUrl)>0:
  552. newPost['cc']=[ccUrl]
  553. if newPost.get('object'):
  554. newPost['object']['cc']=[ccUrl]
  555. # if this is a moderation report then add a status
  556. if isModerationReport:
  557. # add status
  558. if newPost.get('object'):
  559. newPost['object']['moderationStatus']='pending'
  560. else:
  561. newPost['moderationStatus']='pending'
  562. # save to index file
  563. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  564. modFile=open(moderationIndexFile, "a+")
  565. if modFile:
  566. modFile.write(newPostId+'\n')
  567. modFile.close()
  568. if saveToFile:
  569. savePostToBox(baseDir,httpPrefix,newPostId, \
  570. nickname,domain,newPost,'outbox')
  571. return newPost
  572. def outboxMessageCreateWrap(httpPrefix: str, \
  573. nickname: str,domain: str,port: int, \
  574. messageJson: {}) -> {}:
  575. """Wraps a received message in a Create
  576. https://www.w3.org/TR/activitypub/#object-without-create
  577. """
  578. if port:
  579. if port!=80 and port!=443:
  580. if ':' not in domain:
  581. domain=domain+':'+str(port)
  582. statusNumber,published = getStatusNumber()
  583. if messageJson.get('published'):
  584. published = messageJson['published']
  585. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  586. cc=[]
  587. if messageJson.get('cc'):
  588. cc=messageJson['cc']
  589. # TODO
  590. capabilityUrl=[]
  591. newPost = {
  592. "@context": "https://www.w3.org/ns/activitystreams",
  593. 'id': newPostId+'/activity',
  594. 'capability': capabilityUrl,
  595. 'type': 'Create',
  596. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  597. 'published': published,
  598. 'to': messageJson['to'],
  599. 'cc': cc,
  600. 'object': messageJson
  601. }
  602. newPost['object']['id']=newPost['id']
  603. newPost['object']['url']= \
  604. httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
  605. newPost['object']['atomUri']= \
  606. httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  607. return newPost
  608. def postIsAddressedToFollowers(baseDir: str,
  609. nickname: str, domain: str, port: int,httpPrefix: str,
  610. postJsonObject: {}) -> bool:
  611. """Returns true if the given post is addressed to followers of the nickname
  612. """
  613. if port:
  614. if port!=80 and port!=443:
  615. if ':' not in domain:
  616. domain=domain+':'+str(port)
  617. if not postJsonObject.get('object'):
  618. return False
  619. toList=[]
  620. ccList=[]
  621. if postJsonObject['type']!='Update' and \
  622. isinstance(postJsonObject['object'], dict):
  623. if not postJsonObject['object'].get('to'):
  624. return False
  625. toList=postJsonObject['object']['to']
  626. if postJsonObject['object'].get('cc'):
  627. ccList=postJsonObject['object']['cc']
  628. else:
  629. if not postJsonObject.get('to'):
  630. return False
  631. toList=postJsonObject['to']
  632. if postJsonObject.get('cc'):
  633. ccList=postJsonObject['cc']
  634. followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  635. # does the followers url exist in 'to' or 'cc' lists?
  636. addressedToFollowers=False
  637. if followersUrl in toList:
  638. addressedToFollowers=True
  639. if not addressedToFollowers:
  640. if followersUrl in ccList:
  641. addressedToFollowers=True
  642. return addressedToFollowers
  643. def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
  644. """Returns true if the given post is addressed to public
  645. """
  646. if not postJsonObject.get('object'):
  647. return False
  648. if not postJsonObject['object'].get('to'):
  649. return False
  650. publicUrl='https://www.w3.org/ns/activitystreams#Public'
  651. # does the public url exist in 'to' or 'cc' lists?
  652. addressedToPublic=False
  653. if publicUrl in postJsonObject['object']['to']:
  654. addressedToPublic=True
  655. if not addressedToPublic:
  656. if not postJsonObject['object'].get('cc'):
  657. return False
  658. if publicUrl in postJsonObject['object']['cc']:
  659. addressedToPublic=True
  660. return addressedToPublic
  661. def createPublicPost(baseDir: str,
  662. nickname: str, domain: str, port: int,httpPrefix: str, \
  663. content: str, followersOnly: bool, saveToFile: bool,
  664. clientToServer: bool,\
  665. attachImageFilename: str,mediaType: str, \
  666. imageDescription: str,useBlurhash: bool, \
  667. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  668. """Public post
  669. """
  670. domainFull=domain
  671. if port:
  672. if port!=80 and port!=443:
  673. if ':' not in domain:
  674. domainFull=domain+':'+str(port)
  675. return createPostBase(baseDir,nickname, domain, port, \
  676. 'https://www.w3.org/ns/activitystreams#Public', \
  677. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  678. httpPrefix, content, followersOnly, saveToFile, \
  679. clientToServer, \
  680. attachImageFilename,mediaType, \
  681. imageDescription,useBlurhash, \
  682. False,inReplyTo,inReplyToAtomUri,subject)
  683. def createUnlistedPost(baseDir: str,
  684. nickname: str, domain: str, port: int,httpPrefix: str, \
  685. content: str, followersOnly: bool, saveToFile: bool,
  686. clientToServer: bool,\
  687. attachImageFilename: str,mediaType: str, \
  688. imageDescription: str,useBlurhash: bool, \
  689. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  690. """Unlisted post. This has the #Public and followers links inverted.
  691. """
  692. domainFull=domain
  693. if port:
  694. if port!=80 and port!=443:
  695. if ':' not in domain:
  696. domainFull=domain+':'+str(port)
  697. return createPostBase(baseDir,nickname, domain, port, \
  698. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  699. 'https://www.w3.org/ns/activitystreams#Public', \
  700. httpPrefix, content, followersOnly, saveToFile, \
  701. clientToServer, \
  702. attachImageFilename,mediaType, \
  703. imageDescription,useBlurhash, \
  704. False,inReplyTo, inReplyToAtomUri, subject)
  705. def createFollowersOnlyPost(baseDir: str,
  706. nickname: str, domain: str, port: int,httpPrefix: str, \
  707. content: str, followersOnly: bool, saveToFile: bool,
  708. clientToServer: bool,\
  709. attachImageFilename: str,mediaType: str, \
  710. imageDescription: str,useBlurhash: bool, \
  711. inReplyTo=None, inReplyToAtomUri=None, subject=None) -> {}:
  712. """Followers only post
  713. """
  714. domainFull=domain
  715. if port:
  716. if port!=80 and port!=443:
  717. if ':' not in domain:
  718. domainFull=domain+':'+str(port)
  719. return createPostBase(baseDir,nickname, domain, port, \
  720. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  721. None,
  722. httpPrefix, content, followersOnly, saveToFile, \
  723. clientToServer, \
  724. attachImageFilename,mediaType, \
  725. imageDescription,useBlurhash, \
  726. False,inReplyTo, inReplyToAtomUri, subject)
  727. def getMentionedPeople(baseDir: str,httpPrefix: str, \
  728. content: str,domain: str,debug: bool) -> []:
  729. """Extracts a list of mentioned actors from the given message content
  730. """
  731. if '@' not in content:
  732. return None
  733. mentions=[]
  734. words=content.split(' ')
  735. for wrd in words:
  736. if wrd.startswith('@'):
  737. handle=wrd[1:]
  738. if debug:
  739. print('DEBUG: mentioned handle '+handle)
  740. if '@' not in handle:
  741. handle=handle+'@'+domain
  742. if not os.path.isdir(baseDir+'/accounts/'+handle):
  743. continue
  744. else:
  745. externalDomain=handle.split('@')[1]
  746. if not ('.' in externalDomain or externalDomain=='localhost'):
  747. continue
  748. mentionedNickname=handle.split('@')[0]
  749. mentionedDomain=handle.split('@')[1].strip('\n')
  750. if ':' in mentionedDomain:
  751. mentionedDomain=mentionedDomain.split(':')[0]
  752. if not validNickname(mentionedDomain,mentionedNickname):
  753. continue
  754. actor=httpPrefix+'://'+handle.split('@')[1]+'/users/'+mentionedNickname
  755. mentions.append(actor)
  756. return mentions
  757. def createDirectMessagePost(baseDir: str,
  758. nickname: str, domain: str, port: int,httpPrefix: str, \
  759. content: str, followersOnly: bool, saveToFile: bool,
  760. clientToServer: bool,\
  761. attachImageFilename: str,mediaType: str, \
  762. imageDescription: str,useBlurhash: bool, \
  763. inReplyTo=None, inReplyToAtomUri=None, subject=None,debug=False) -> {}:
  764. """Direct Message post
  765. """
  766. mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
  767. if debug:
  768. print('mentionedPeople: '+str(mentionedPeople))
  769. if not mentionedPeople:
  770. return None
  771. postTo=None
  772. postCc=None
  773. return createPostBase(baseDir,nickname, domain, port, \
  774. postTo,postCc, \
  775. httpPrefix, content, followersOnly, saveToFile, \
  776. clientToServer, \
  777. attachImageFilename,mediaType, \
  778. imageDescription,useBlurhash, \
  779. False,inReplyTo, inReplyToAtomUri, subject)
  780. def createReportPost(baseDir: str,
  781. nickname: str, domain: str, port: int,httpPrefix: str, \
  782. content: str, followersOnly: bool, saveToFile: bool,
  783. clientToServer: bool,\
  784. attachImageFilename: str,mediaType: str, \
  785. imageDescription: str,useBlurhash: bool, \
  786. debug: bool,subject=None) -> {}:
  787. """Send a report to moderators
  788. """
  789. domainFull=domain
  790. if port:
  791. if port!=80 and port!=443:
  792. if ':' not in domain:
  793. domainFull=domain+':'+str(port)
  794. # add a title to distinguish moderation reports from other posts
  795. reportTitle='Moderation Report'
  796. if not subject:
  797. subject=reportTitle
  798. else:
  799. if not subject.startswith(reportTitle):
  800. subject=reportTitle+': '+subject
  801. # create the list of moderators from the moderators file
  802. moderatorsList=[]
  803. moderatorsFile=baseDir+'/accounts/moderators.txt'
  804. if os.path.isfile(moderatorsFile):
  805. with open (moderatorsFile, "r") as fileHandler:
  806. for line in fileHandler:
  807. line=line.strip('\n')
  808. if line.startswith('#'):
  809. continue
  810. if line.startswith('/users/'):
  811. line=line.replace('users','')
  812. if line.startswith('@'):
  813. line=line[1:]
  814. if '@' in line:
  815. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
  816. if moderatorActor not in moderatorList:
  817. moderatorsList.append(moderatorActor)
  818. continue
  819. if line.startswith('http') or line.startswith('dat'):
  820. # must be a local address - no remote moderators
  821. if '://'+domainFull+'/' in line:
  822. if line not in moderatorsList:
  823. moderatorsList.append(line)
  824. else:
  825. if '/' not in line:
  826. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
  827. if moderatorActor not in moderatorsList:
  828. moderatorsList.append(moderatorActor)
  829. if len(moderatorsList)==0:
  830. # if there are no moderators then the admin becomes the moderator
  831. adminNickname=getConfigParam(baseDir,'admin')
  832. if adminNickname:
  833. moderatorsList.append(httpPrefix+'://'+domainFull+'/users/'+adminNickname)
  834. if not moderatorsList:
  835. return None
  836. if debug:
  837. print('DEBUG: Sending report to moderators')
  838. print(str(moderatorsList))
  839. postTo=moderatorsList
  840. postCc=None
  841. postJsonObject=None
  842. for toUrl in postTo:
  843. postJsonObject= \
  844. createPostBase(baseDir,nickname, domain, port, \
  845. toUrl,postCc, \
  846. httpPrefix, content, followersOnly, saveToFile, \
  847. clientToServer, \
  848. attachImageFilename,mediaType, \
  849. imageDescription,useBlurhash, \
  850. True,None, None, subject)
  851. return postJsonObject
  852. def threadSendPost(session,postJsonStr: str,federationList: [],\
  853. inboxUrl: str, baseDir: str,signatureHeaderJson: {},postLog: [],
  854. debug :bool) -> None:
  855. """Sends a post with exponential backoff
  856. """
  857. tries=0
  858. backoffTime=60
  859. for attempt in range(20):
  860. postResult = \
  861. postJsonString(session,postJsonStr,federationList, \
  862. inboxUrl,signatureHeaderJson, \
  863. "inbox:write",debug)
  864. if postResult:
  865. logStr='Success on try '+str(tries)+': '+postJsonStr
  866. else:
  867. logStr='Retry '+str(tries)+': '+postJsonStr
  868. postLog.append(logStr)
  869. # keep the length of the log finite
  870. # Don't accumulate massive files on systems with limited resources
  871. while len(postLog)>16:
  872. postLog.pop(0)
  873. # save the log file
  874. postLogFilename=baseDir+'/post.log'
  875. with open(postLogFilename, "a+") as logFile:
  876. logFile.write(logStr+'\n')
  877. if postResult:
  878. if debug:
  879. print('DEBUG: successful json post to '+inboxUrl)
  880. # our work here is done
  881. break
  882. if debug:
  883. print(postJsonStr)
  884. print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
  885. str(backoffTime)+' seconds.')
  886. time.sleep(backoffTime)
  887. backoffTime *= 2
  888. tries+=1
  889. def sendPost(projectVersion: str, \
  890. session,baseDir: str,nickname: str, domain: str, port: int, \
  891. toNickname: str, toDomain: str, toPort: int, cc: str, \
  892. httpPrefix: str, content: str, followersOnly: bool, \
  893. saveToFile: bool, clientToServer: bool, \
  894. attachImageFilename: str,mediaType: str, \
  895. imageDescription: str,useBlurhash: bool, \
  896. federationList: [],\
  897. sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
  898. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  899. """Post to another inbox
  900. """
  901. withDigest=True
  902. if toNickname=='inbox':
  903. # shared inbox actor on @domain@domain
  904. toNickname=toDomain
  905. toDomainOriginal=toDomain
  906. if toPort:
  907. if toPort!=80 and toPort!=443:
  908. if ':' not in toDomain:
  909. toDomain=toDomain+':'+str(toPort)
  910. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  911. # lookup the inbox for the To handle
  912. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  913. domain,projectVersion)
  914. if not wfRequest:
  915. return 1
  916. if not clientToServer:
  917. postToBox='inbox'
  918. else:
  919. postToBox='outbox'
  920. # get the actor inbox for the To handle
  921. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  922. getPersonBox(baseDir,session,wfRequest,personCache, \
  923. projectVersion,httpPrefix,domain,postToBox)
  924. # If there are more than one followers on the target domain
  925. # then send to the shared inbox indead of the individual inbox
  926. if nickname=='capabilities':
  927. inboxUrl=capabilityAcquisition
  928. if not capabilityAcquisition:
  929. return 2
  930. if not inboxUrl:
  931. return 3
  932. if not pubKey:
  933. return 4
  934. if not toPersonId:
  935. return 5
  936. # sharedInbox and capabilities are optional
  937. postJsonObject = \
  938. createPostBase(baseDir,nickname,domain,port, \
  939. toPersonId,cc,httpPrefix,content, \
  940. followersOnly,saveToFile,clientToServer, \
  941. attachImageFilename,mediaType, \
  942. imageDescription,useBlurhash, \
  943. False,inReplyTo,inReplyToAtomUri,subject)
  944. # get the senders private key
  945. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
  946. if len(privateKeyPem)==0:
  947. return 6
  948. if toDomain not in inboxUrl:
  949. return 7
  950. postPath=inboxUrl.split(toDomain,1)[1]
  951. # convert json to string so that there are no
  952. # subsequent conversions after creating message body digest
  953. postJsonStr=json.dumps(postJsonObject)
  954. # construct the http header, including the message body digest
  955. signatureHeaderJson = \
  956. createSignedHeader(privateKeyPem,nickname,domain,port, \
  957. toDomain,toPort, \
  958. postPath,httpPrefix,withDigest,postJsonStr)
  959. # Keep the number of threads being used small
  960. while len(sendThreads)>10:
  961. sendThreads[0].kill()
  962. sendThreads.pop(0)
  963. thr = threadWithTrace(target=threadSendPost,args=(session, \
  964. postJsonStr, \
  965. federationList, \
  966. inboxUrl,baseDir, \
  967. signatureHeaderJson.copy(), \
  968. postLog,
  969. debug),daemon=True)
  970. sendThreads.append(thr)
  971. thr.start()
  972. return 0
  973. def sendPostViaServer(projectVersion: str, \
  974. baseDir: str,session,fromNickname: str,password: str, \
  975. fromDomain: str, fromPort: int, \
  976. toNickname: str, toDomain: str, toPort: int, cc: str, \
  977. httpPrefix: str, content: str, followersOnly: bool, \
  978. attachImageFilename: str,mediaType: str, \
  979. imageDescription: str,useBlurhash: bool, \
  980. cachedWebfingers: {},personCache: {}, \
  981. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  982. """Send a post via a proxy (c2s)
  983. """
  984. if not session:
  985. print('WARN: No session for sendPostViaServer')
  986. return 6
  987. withDigest=True
  988. if toPort:
  989. if toPort!=80 and toPort!=443:
  990. if ':' not in fromDomain:
  991. fromDomain=fromDomain+':'+str(fromPort)
  992. handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
  993. # lookup the inbox for the To handle
  994. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  995. fromDomain,projectVersion)
  996. if not wfRequest:
  997. if debug:
  998. print('DEBUG: webfinger failed for '+handle)
  999. return 1
  1000. postToBox='outbox'
  1001. # get the actor inbox for the To handle
  1002. inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1003. getPersonBox(baseDir,session,wfRequest,personCache, \
  1004. projectVersion,httpPrefix,fromDomain,postToBox)
  1005. if not inboxUrl:
  1006. if debug:
  1007. print('DEBUG: No '+postToBox+' was found for '+handle)
  1008. return 3
  1009. if not fromPersonId:
  1010. if debug:
  1011. print('DEBUG: No actor was found for '+handle)
  1012. return 4
  1013. # Get the json for the c2s post, not saving anything to file
  1014. # Note that baseDir is set to None
  1015. saveToFile=False
  1016. clientToServer=True
  1017. if toDomain.lower().endswith('public'):
  1018. toPersonId='https://www.w3.org/ns/activitystreams#Public'
  1019. fromDomainFull=fromDomain
  1020. if fromPort:
  1021. if fromPort!=80 and fromPort!=443:
  1022. if ':' not in fromDomain:
  1023. fromDomainFull=fromDomain+':'+str(fromPort)
  1024. cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1025. else:
  1026. if toDomain.lower().endswith('followers') or \
  1027. toDomain.lower().endswith('followersonly'):
  1028. toPersonId=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1029. else:
  1030. toDomainFull=toDomain
  1031. if toPort:
  1032. if toPort!=80 and toPort!=443:
  1033. if ':' not in toDomain:
  1034. toDomainFull=toDomain+':'+str(toPort)
  1035. toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
  1036. postJsonObject = \
  1037. createPostBase(baseDir, \
  1038. fromNickname,fromDomain,fromPort, \
  1039. toPersonId,cc,httpPrefix,content, \
  1040. followersOnly,saveToFile,clientToServer, \
  1041. attachImageFilename,mediaType, \
  1042. imageDescription,useBlurhash, \
  1043. False,inReplyTo,inReplyToAtomUri,subject)
  1044. authHeader=createBasicAuthHeader(fromNickname,password)
  1045. if attachImageFilename:
  1046. headers = {'host': fromDomain, \
  1047. 'Authorization': authHeader}
  1048. postResult = \
  1049. postImage(session,attachImageFilename,[],inboxUrl,headers,"inbox:write")
  1050. #if not postResult:
  1051. # if debug:
  1052. # print('DEBUG: Failed to upload image')
  1053. # return 9
  1054. headers = {'host': fromDomain, \
  1055. 'Content-type': 'application/json', \
  1056. 'Authorization': authHeader}
  1057. postResult = \
  1058. postJsonString(session,json.dumps(postJsonObject),[],inboxUrl,headers,"inbox:write",debug)
  1059. #if not postResult:
  1060. # if debug:
  1061. # print('DEBUG: POST failed for c2s to '+inboxUrl)
  1062. # return 5
  1063. if debug:
  1064. print('DEBUG: c2s POST success')
  1065. return 0
  1066. def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
  1067. """Returns a dictionary with followers grouped by domain
  1068. """
  1069. handle=nickname+'@'+domain
  1070. followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
  1071. if not os.path.isfile(followersFilename):
  1072. return None
  1073. grouped={}
  1074. with open(followersFilename, "r") as f:
  1075. for followerHandle in f:
  1076. if '@' in followerHandle:
  1077. fHandle=followerHandle.strip().replace('\n','')
  1078. followerDomain=fHandle.split('@')[1]
  1079. if not grouped.get(followerDomain):
  1080. grouped[followerDomain]=[fHandle]
  1081. else:
  1082. grouped[followerDomain].append(fHandle)
  1083. return grouped
  1084. def sendSignedJson(postJsonObject: {},session,baseDir: str, \
  1085. nickname: str, domain: str, port: int, \
  1086. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1087. httpPrefix: str, saveToFile: bool, clientToServer: bool, \
  1088. federationList: [], \
  1089. sendThreads: [], postLog: [], cachedWebfingers: {}, \
  1090. personCache: {}, debug: bool,projectVersion: str) -> int:
  1091. """Sends a signed json object to an inbox/outbox
  1092. """
  1093. if debug:
  1094. print('DEBUG: sendSignedJson start')
  1095. if not session:
  1096. print('WARN: No session specified for sendSignedJson')
  1097. return 8
  1098. withDigest=True
  1099. sharedInbox=False
  1100. if toNickname=='inbox':
  1101. # shared inbox actor on @domain@domain
  1102. toNickname=toDomain
  1103. sharedInbox=True
  1104. toDomainOriginal=toDomain
  1105. if toPort:
  1106. if toPort!=80 and toPort!=443:
  1107. if ':' not in toDomain:
  1108. toDomain=toDomain+':'+str(toPort)
  1109. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  1110. if debug:
  1111. print('DEBUG: handle - '+handle+' toPort '+str(toPort))
  1112. # lookup the inbox for the To handle
  1113. wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1114. domain,projectVersion)
  1115. if not wfRequest:
  1116. if debug:
  1117. print('DEBUG: webfinger for '+handle+' failed')
  1118. return 1
  1119. if not clientToServer:
  1120. postToBox='inbox'
  1121. else:
  1122. postToBox='outbox'
  1123. # get the actor inbox/outbox/capabilities for the To handle
  1124. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
  1125. getPersonBox(baseDir,session,wfRequest,personCache, \
  1126. projectVersion,httpPrefix,domain,postToBox)
  1127. if nickname=='capabilities':
  1128. inboxUrl=capabilityAcquisition
  1129. if not capabilityAcquisition:
  1130. return 2
  1131. else:
  1132. print("inboxUrl: "+str(inboxUrl))
  1133. print("toPersonId: "+str(toPersonId))
  1134. print("sharedInboxUrl: "+str(sharedInboxUrl))
  1135. if inboxUrl:
  1136. if inboxUrl.endswith('/actor/inbox'):
  1137. inboxUrl=sharedInboxUrl
  1138. if not inboxUrl:
  1139. if debug:
  1140. print('DEBUG: missing inboxUrl')
  1141. return 3
  1142. if debug:
  1143. print('DEBUG: Sending to endpoint '+inboxUrl)
  1144. if not pubKey:
  1145. if debug:
  1146. print('DEBUG: missing pubkey')
  1147. return 4
  1148. if not toPersonId:
  1149. if debug:
  1150. print('DEBUG: missing personId')
  1151. return 5
  1152. # sharedInbox and capabilities are optional
  1153. # get the senders private key
  1154. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
  1155. if len(privateKeyPem)==0:
  1156. if debug:
  1157. print('DEBUG: Private key not found for '+nickname+'@'+domain+' in '+baseDir+'/keys/private')
  1158. return 6
  1159. if toDomain not in inboxUrl:
  1160. if debug:
  1161. print('DEBUG: '+toDomain+' is not in '+inboxUrl)
  1162. return 7
  1163. postPath=inboxUrl.split(toDomain,1)[1]
  1164. # convert json to string so that there are no
  1165. # subsequent conversions after creating message body digest
  1166. postJsonStr=json.dumps(postJsonObject)
  1167. # construct the http header, including the message body digest
  1168. signatureHeaderJson = \
  1169. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1170. toDomain,toPort, \
  1171. postPath,httpPrefix,withDigest,postJsonStr)
  1172. # Keep the number of threads being used small
  1173. while len(sendThreads)>10:
  1174. sendThreads[0].kill()
  1175. sendThreads.pop(0)
  1176. if debug:
  1177. print('DEBUG: starting thread to send post')
  1178. pprint(postJsonObject)
  1179. thr = threadWithTrace(target=threadSendPost, \
  1180. args=(session, \
  1181. postJsonStr, \
  1182. federationList, \
  1183. inboxUrl,baseDir, \
  1184. signatureHeaderJson.copy(), \
  1185. postLog,
  1186. debug),daemon=True)
  1187. sendThreads.append(thr)
  1188. thr.start()
  1189. return 0
  1190. def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
  1191. """The Follow activity doesn't have a 'to' field and so one
  1192. needs to be added so that activity distribution happens in a consistent way
  1193. Returns true if a 'to' field exists or was added
  1194. """
  1195. if postJsonObject.get('to'):
  1196. return postJsonObject,True
  1197. if debug:
  1198. pprint(postJsonObject)
  1199. print('DEBUG: no "to" field when sending to named addresses 2')
  1200. isSameType=False
  1201. toFieldAdded=False
  1202. if postJsonObject.get('object'):
  1203. if isinstance(postJsonObject['object'], str):
  1204. if postJsonObject.get('type'):
  1205. if postJsonObject['type']==activityType:
  1206. isSameType=True
  1207. if debug:
  1208. print('DEBUG: "to" field assigned to Follow')
  1209. toAddress=postJsonObject['object']
  1210. if '/statuses/' in toAddress:
  1211. toAddress=toAddress.split('/statuses/')[0]
  1212. postJsonObject['to']=[toAddress]
  1213. toFieldAdded=True
  1214. elif isinstance(postJsonObject['object'], dict):
  1215. if postJsonObject['object'].get('type'):
  1216. if postJsonObject['object']['type']==activityType:
  1217. isSameType=True
  1218. if isinstance(postJsonObject['object']['object'], str):
  1219. if debug:
  1220. print('DEBUG: "to" field assigned to Follow')
  1221. toAddress=postJsonObject['object']['object']
  1222. if '/statuses/' in toAddress:
  1223. toAddress=toAddress.split('/statuses/')[0]
  1224. postJsonObject['object']['to']=[toAddress]
  1225. postJsonObject['to']=[postJsonObject['object']['object']]
  1226. toFieldAdded=True
  1227. if not isSameType:
  1228. return postJsonObject,True
  1229. if toFieldAdded:
  1230. return postJsonObject,True
  1231. return postJsonObject,False
  1232. def sendToNamedAddresses(session,baseDir: str, \
  1233. nickname: str, domain: str, port: int, \
  1234. httpPrefix: str,federationList: [], \
  1235. sendThreads: [],postLog: [], \
  1236. cachedWebfingers: {},personCache: {}, \
  1237. postJsonObject: {},debug: bool, \
  1238. projectVersion: str) -> None:
  1239. """sends a post to the specific named addresses in to/cc
  1240. """
  1241. if not session:
  1242. print('WARN: No session for sendToNamedAddresses')
  1243. return
  1244. if not postJsonObject.get('object'):
  1245. return
  1246. if isinstance(postJsonObject['object'], dict):
  1247. isProfileUpdate=False
  1248. # for actor updates there is no 'to' within the object
  1249. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1250. if postJsonObject['type']=='Update' and \
  1251. (postJsonObject['object']['type']=='Person' or \
  1252. postJsonObject['object']['type']=='Application' or \
  1253. postJsonObject['object']['type']=='Service'):
  1254. # use the original object, which has a 'to'
  1255. recipientsObject=postJsonObject
  1256. isProfileUpdate=True
  1257. if not isProfileUpdate:
  1258. if not postJsonObject['object'].get('to'):
  1259. if debug:
  1260. pprint(postJsonObject)
  1261. print('DEBUG: no "to" field when sending to named addresses')
  1262. if postJsonObject['object'].get('type'):
  1263. if postJsonObject['object']['type']=='Follow':
  1264. if isinstance(postJsonObject['object']['object'], str):
  1265. if debug:
  1266. print('DEBUG: "to" field assigned to Follow')
  1267. postJsonObject['object']['to']=[postJsonObject['object']['object']]
  1268. if not postJsonObject['object'].get('to'):
  1269. return
  1270. recipientsObject=postJsonObject['object']
  1271. else:
  1272. postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
  1273. if not fieldAdded:
  1274. return
  1275. postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
  1276. if not fieldAdded:
  1277. return
  1278. recipientsObject=postJsonObject
  1279. recipients=[]
  1280. recipientType=['to','cc']
  1281. for rType in recipientType:
  1282. if not recipientsObject.get(rType):
  1283. continue
  1284. if isinstance(recipientsObject[rType], list):
  1285. if debug:
  1286. pprint(recipientsObject)
  1287. print('recipientsObject: '+str(recipientsObject[rType]))
  1288. for address in recipientsObject[rType]:
  1289. if not address:
  1290. continue
  1291. if '/' not in address:
  1292. continue
  1293. if address.endswith('#Public'):
  1294. continue
  1295. if address.endswith('/followers'):
  1296. continue
  1297. recipients.append(address)
  1298. elif isinstance(recipientsObject[rType], str):
  1299. address=recipientsObject[rType]
  1300. if address:
  1301. if '/' in address:
  1302. if address.endswith('#Public'):
  1303. continue
  1304. if address.endswith('/followers'):
  1305. continue
  1306. recipients.append(address)
  1307. if not recipients:
  1308. if debug:
  1309. print('DEBUG: no individual recipients')
  1310. return
  1311. if debug:
  1312. print('DEBUG: Sending individually addressed posts: '+str(recipients))
  1313. # this is after the message has arrived at the server
  1314. clientToServer=False
  1315. for address in recipients:
  1316. toNickname=getNicknameFromActor(address)
  1317. if not toNickname:
  1318. continue
  1319. toDomain,toPort=getDomainFromActor(address)
  1320. if not toDomain:
  1321. continue
  1322. if debug:
  1323. domainFull=domain
  1324. if port:
  1325. if port!=80 and port!=443:
  1326. if ':' not in domain:
  1327. domainFull=domain+':'+str(port)
  1328. toDomainFull=toDomain
  1329. if toPort:
  1330. if toPort!=80 and toPort!=443:
  1331. if ':' not in toDomain:
  1332. toDomainFull=toDomain+':'+str(toPort)
  1333. print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+' to '+toNickname+'@'+toDomainFull)
  1334. cc=[]
  1335. sendSignedJson(postJsonObject,session,baseDir, \
  1336. nickname,domain,port, \
  1337. toNickname,toDomain,toPort, \
  1338. cc,httpPrefix,True,clientToServer, \
  1339. federationList, \
  1340. sendThreads,postLog,cachedWebfingers, \
  1341. personCache,debug,projectVersion)
  1342. def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
  1343. """Returns true if the given domain has a shared inbox
  1344. """
  1345. wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
  1346. None,__version__)
  1347. if wfRequest:
  1348. return True
  1349. return False
  1350. def sendToFollowers(session,baseDir: str, \
  1351. nickname: str, domain: str, port: int, \
  1352. httpPrefix: str,federationList: [], \
  1353. sendThreads: [],postLog: [], \
  1354. cachedWebfingers: {},personCache: {}, \
  1355. postJsonObject: {},debug: bool, \
  1356. projectVersion: str) -> None:
  1357. """sends a post to the followers of the given nickname
  1358. """
  1359. print('sendToFollowers')
  1360. if not session:
  1361. print('WARN: No session for sendToFollowers')
  1362. return
  1363. if not postIsAddressedToFollowers(baseDir,nickname,domain, \
  1364. port,httpPrefix,postJsonObject):
  1365. if debug:
  1366. print('Post is not addressed to followers')
  1367. return
  1368. print('Post is addressed to followers')
  1369. grouped=groupFollowersByDomain(baseDir,nickname,domain)
  1370. if not grouped:
  1371. if debug:
  1372. print('Post to followers did not resolve any domains')
  1373. return
  1374. print('Post to followers resolved domains')
  1375. print(str(grouped))
  1376. # this is after the message has arrived at the server
  1377. clientToServer=False
  1378. # for each instance
  1379. for followerDomain,followerHandles in grouped.items():
  1380. if debug:
  1381. print('DEBUG: follower handles for '+followerDomain)
  1382. pprint(followerHandles)
  1383. withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
  1384. if debug:
  1385. if withSharedInbox:
  1386. print(followerDomain+' has shared inbox')
  1387. else:
  1388. print(followerDomain+' does not have a shared inbox')
  1389. toPort=port
  1390. index=0
  1391. toDomain=followerHandles[index].split('@')[1]
  1392. if ':' in toDomain:
  1393. toPort=toDomain.split(':')[1]
  1394. toDomain=toDomain.split(':')[0]
  1395. # If this is a profile update then send to shared inbox
  1396. cc=''
  1397. if withSharedInbox:
  1398. toNickname=followerHandles[index].split('@')[0]
  1399. # if there are more than one followers on the domain
  1400. # then send the post to the shared inbox
  1401. if len(followerHandles)>1:
  1402. toNickname='inbox'
  1403. if toNickname!='inbox' and postJsonObject.get('type'):
  1404. if postJsonObject['type']=='Update':
  1405. if postJsonObject.get('object'):
  1406. if isinstance(postJsonObject['object'], dict):
  1407. if postJsonObject['object'].get('type'):
  1408. if postJsonObject['object']['type']=='Person' or \
  1409. postJsonObject['object']['type']=='Application' or \
  1410. postJsonObject['object']['type']=='Service':
  1411. print('Sending profile update to shared inbox of '+toDomain)
  1412. toNickname='inbox'
  1413. if debug:
  1414. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1415. sendSignedJson(postJsonObject,session,baseDir, \
  1416. nickname,domain,port, \
  1417. toNickname,toDomain,toPort, \
  1418. cc,httpPrefix,True,clientToServer, \
  1419. federationList, \
  1420. sendThreads,postLog,cachedWebfingers, \
  1421. personCache,debug,projectVersion)
  1422. else:
  1423. # send to individual followers without using a shared inbox
  1424. for handle in followerHandles:
  1425. toNickname=handle.split('@')[0]
  1426. if debug:
  1427. print('DEBUG: Sending from '+nickname+'@'+domain+' to '+toNickname+'@'+toDomain)
  1428. sendSignedJson(postJsonObject,session,baseDir, \
  1429. nickname,domain,port, \
  1430. toNickname,toDomain,toPort, \
  1431. cc,httpPrefix,True,clientToServer, \
  1432. federationList, \
  1433. sendThreads,postLog,cachedWebfingers, \
  1434. personCache,debug,projectVersion)
  1435. if debug:
  1436. print('DEBUG: End of sendToFollowers')
  1437. def createInbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1438. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1439. return createBoxBase(baseDir,'inbox',nickname,domain,port,httpPrefix, \
  1440. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1441. def createDMTimeline(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1442. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1443. return createBoxBase(baseDir,'dm',nickname,domain,port,httpPrefix, \
  1444. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1445. def createOutbox(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1446. itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
  1447. return createBoxBase(baseDir,'outbox',nickname,domain,port,httpPrefix, \
  1448. itemsPerPage,headerOnly,authorized,False,pageNumber)
  1449. def createModeration(baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1450. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1451. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1452. boxname='moderation'
  1453. if port:
  1454. if port!=80 and port!=443:
  1455. if ':' not in domain:
  1456. domain=domain+':'+str(port)
  1457. if not pageNumber:
  1458. pageNumber=1
  1459. pageStr='?page='+str(pageNumber)
  1460. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1461. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1462. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1463. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1464. 'totalItems': 0,
  1465. 'type': 'OrderedCollection'}
  1466. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1467. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1468. 'orderedItems': [
  1469. ],
  1470. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1471. 'type': 'OrderedCollectionPage'}
  1472. if isModerator(baseDir,nickname):
  1473. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  1474. if os.path.isfile(moderationIndexFile):
  1475. with open(moderationIndexFile, "r") as f:
  1476. lines = f.readlines()
  1477. boxHeader['totalItems']=len(lines)
  1478. if headerOnly:
  1479. return boxHeader
  1480. pageLines=[]
  1481. if len(lines)>0:
  1482. endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
  1483. if endLineNumber<0:
  1484. endLineNumber=0
  1485. startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
  1486. if startLineNumber<0:
  1487. startLineNumber=0
  1488. lineNumber=startLineNumber
  1489. while lineNumber>=endLineNumber:
  1490. pageLines.append(lines[lineNumber].strip('\n'))
  1491. lineNumber-=1
  1492. for postUrl in pageLines:
  1493. postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
  1494. if os.path.isfile(postFilename):
  1495. with open(postFilename, 'r') as fp:
  1496. postJsonObject=commentjson.load(fp)
  1497. boxItems['orderedItems'].append(postJsonObject)
  1498. if headerOnly:
  1499. return boxHeader
  1500. return boxItems
  1501. def getStatusNumberFromPostFilename(filename) -> int:
  1502. """Gets the status number from a post filename
  1503. eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
  1504. returns 156295850695206
  1505. """
  1506. if '#statuses#' not in filename:
  1507. return None
  1508. return int(filename.split('#')[-1].replace('.json',''))
  1509. def isDM(postJsonObject: {}) -> bool:
  1510. """Returns true if the given post is a DM
  1511. """
  1512. if postJsonObject['type']!='Create':
  1513. return False
  1514. if not postJsonObject.get('object'):
  1515. return False
  1516. if not isinstance(postJsonObject['object'], dict):
  1517. return False
  1518. if postJsonObject['object']['type']!='Note':
  1519. return False
  1520. fields=['to','cc']
  1521. for f in fields:
  1522. if not postJsonObject['object'].get(f):
  1523. continue
  1524. for toAddress in postJsonObject['object'][f]:
  1525. if toAddress.endswith('#Public'):
  1526. return False
  1527. if toAddress.endswith('followers'):
  1528. return False
  1529. return True
  1530. def createBoxBase(baseDir: str,boxname: str, \
  1531. nickname: str,domain: str,port: int,httpPrefix: str, \
  1532. itemsPerPage: int,headerOnly: bool,authorized :bool, \
  1533. ocapAlways: bool,pageNumber=None) -> {}:
  1534. """Constructs the box feed for a person with the given nickname
  1535. """
  1536. if boxname!='inbox' and boxname!='dm' and boxname!='outbox':
  1537. return None
  1538. if boxname!='dm':
  1539. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1540. else:
  1541. # extract DMs from the inbox
  1542. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1543. sharedBoxDir=None
  1544. if boxname=='inbox':
  1545. sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
  1546. if port:
  1547. if port!=80 and port!=443:
  1548. if ':' not in domain:
  1549. domain=domain+':'+str(port)
  1550. pageStr='?page=true'
  1551. if pageNumber:
  1552. try:
  1553. pageStr='?page='+str(pageNumber)
  1554. except:
  1555. pass
  1556. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1557. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1558. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1559. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1560. 'totalItems': 0,
  1561. 'type': 'OrderedCollection'}
  1562. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1563. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1564. 'orderedItems': [
  1565. ],
  1566. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1567. 'type': 'OrderedCollectionPage'}
  1568. # counter for posts so far added to the target page
  1569. postsOnPageCtr=0
  1570. # post filenames sorted in descending order
  1571. postsInBoxDict={}
  1572. postsCtr=0
  1573. postsInPersonInbox=os.listdir(boxDir)
  1574. for postFilename in postsInPersonInbox:
  1575. if not postFilename.endswith('.json'):
  1576. continue
  1577. # extract the status number
  1578. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1579. if statusNumber:
  1580. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1581. postsCtr+=1
  1582. # combine the inbox for the account with the shared inbox
  1583. if sharedBoxDir:
  1584. handle=nickname+'@'+domain
  1585. followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
  1586. postsInSharedInbox=os.listdir(sharedBoxDir)
  1587. for postFilename in postsInSharedInbox:
  1588. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1589. if statusNumber:
  1590. sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
  1591. # get the actor from the shared post
  1592. with open(sharedInboxFilename, 'r') as fp:
  1593. postJsonObject=commentjson.load(fp)
  1594. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  1595. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  1596. if actorNickname and actorDomain:
  1597. # is the actor followed by this account?
  1598. if actorNickname+'@'+actorDomain in open(followingFilename).read():
  1599. if ocapAlways:
  1600. capsList=None
  1601. # Note: should this be in the Create or the object of a post?
  1602. if postJsonObject.get('capability'):
  1603. if isinstance(postJsonObject['capability'], list):
  1604. capsList=postJsonObject['capability']
  1605. # Have capabilities been granted for the sender?
  1606. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/granted/'+postJsonObject['actor'].replace('/','#')+'.json'
  1607. if os.path.isfile(ocapFilename):
  1608. # read the capabilities id
  1609. with open(ocapFilename, 'r') as fp:
  1610. ocapJson=commentjson.load(fp)
  1611. if ocapJson.get('id'):
  1612. if ocapJson['id'] in capsList:
  1613. postsInBoxDict[statusNumber]=sharedInboxFilename
  1614. postsCtr+=1
  1615. else:
  1616. postsInBoxDict[statusNumber]=sharedInboxFilename
  1617. postsCtr+=1
  1618. # sort the list in descending order of date
  1619. postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=True))
  1620. # number of posts in box
  1621. boxHeader['totalItems']=postsCtr
  1622. prevPostFilename=None
  1623. if not pageNumber:
  1624. pageNumber=1
  1625. # Generate first and last entries within header
  1626. if postsCtr>0:
  1627. lastPage=int(postsCtr/itemsPerPage)
  1628. if lastPage<1:
  1629. lastPage=1
  1630. boxHeader['last']= \
  1631. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
  1632. # Insert posts
  1633. currPage=1
  1634. postsCtr=0
  1635. for statusNumber,postFilename in postsInBox.items():
  1636. # Are we at the starting page yet?
  1637. if prevPostFilename and currPage==pageNumber and postsCtr==0:
  1638. # update the prev entry for the last message id
  1639. postId = prevPostFilename.split('#statuses#')[1].replace('#activity','')
  1640. boxHeader['prev']= \
  1641. httpPrefix+'://'+domain+'/users/'+nickname+'/'+ \
  1642. boxname+'?min_id='+postId+'&page=true'
  1643. # get the full path of the post file
  1644. filePath = postFilename
  1645. try:
  1646. if os.path.isfile(filePath):
  1647. # is this a valid timeline post?
  1648. isPost=False
  1649. # must be a "Note" or "Announce" type
  1650. with open(filePath, 'r') as file:
  1651. postStr = file.read()
  1652. if '"Note"' in postStr or '"Announce"' in postStr or \
  1653. ('"Question"' in postStr and '"Create"' in postStr):
  1654. isPost=True
  1655. if boxname=='dm':
  1656. if '#Public' in postStr or '/followers' in postStr:
  1657. isPost=False
  1658. if isPost:
  1659. if currPage == pageNumber and postsOnPageCtr <= itemsPerPage:
  1660. # get the post as json
  1661. p = json.loads(postStr)
  1662. if boxname!='dm' or \
  1663. (boxname=='dm' and isDM(p)):
  1664. # remove any capability so that it's not displayed
  1665. if p.get('capability'):
  1666. del p['capability']
  1667. # Don't show likes or replies to unauthorized viewers
  1668. if not authorized:
  1669. if p.get('object'):
  1670. if isinstance(p['object'], dict):
  1671. if p['object'].get('likes'):
  1672. p['likes']={}
  1673. if p['object'].get('replies'):
  1674. p['replies']={}
  1675. # insert it into the box feed
  1676. if postsOnPageCtr < itemsPerPage:
  1677. if not headerOnly:
  1678. boxItems['orderedItems'].append(p)
  1679. postsOnPageCtr += 1
  1680. elif postsOnPageCtr == itemsPerPage:
  1681. # if this is the last post update the next message ID
  1682. if '/statuses/' in p['id']:
  1683. postId = p['id'].split('/statuses/')[1].replace('/activity','')
  1684. boxHeader['next']= \
  1685. httpPrefix+'://'+domain+'/users/'+ \
  1686. nickname+'/'+boxname+'?max_id='+ \
  1687. postId+'&page=true'
  1688. # remember the last post filename for use with prev
  1689. prevPostFilename = postFilename
  1690. if postsOnPageCtr >= itemsPerPage:
  1691. break
  1692. # count the pages
  1693. postsCtr += 1
  1694. if postsCtr >= itemsPerPage:
  1695. postsCtr = 0
  1696. currPage += 1
  1697. except Exception as e:
  1698. print(e)
  1699. if headerOnly:
  1700. return boxHeader
  1701. return boxItems
  1702. def expireCache(baseDir: str,personCache: {},httpPrefix: str,archiveDir: str,maxPostsInBox=1024):
  1703. """Thread used to expire actors from the cache and archive old posts
  1704. """
  1705. while True:
  1706. # once per day
  1707. time.sleep(60*60*24)
  1708. expirePersonCache(basedir,personCache)
  1709. archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
  1710. def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str,maxPostsInBox=1024) -> None:
  1711. """Archives posts for all accounts
  1712. """
  1713. if archiveDir:
  1714. if not os.path.isdir(archiveDir):
  1715. os.mkdir(archiveDir)
  1716. if archiveDir:
  1717. if not os.path.isdir(archiveDir+'/accounts'):
  1718. os.mkdir(archiveDir+'/accounts')
  1719. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  1720. for handle in dirs:
  1721. if '@' in handle:
  1722. nickname=handle.split('@')[0]
  1723. domain=handle.split('@')[1]
  1724. archiveSubdir=None
  1725. if archiveDir:
  1726. if not os.path.isdir(archiveDir+'/accounts/'+handle):
  1727. os.mkdir(archiveDir+'/accounts/'+handle)
  1728. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
  1729. os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
  1730. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
  1731. os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
  1732. archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
  1733. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  1734. 'inbox',archiveSubdir, \
  1735. maxPostsInBox)
  1736. if archiveDir:
  1737. archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
  1738. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  1739. 'outbox',archiveSubdir, \
  1740. maxPostsInBox)
  1741. def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
  1742. boxname: str,archiveDir: str,maxPostsInBox=1024) -> None:
  1743. """Retain a maximum number of posts within the given box
  1744. Move any others to an archive directory
  1745. """
  1746. if boxname!='inbox' and boxname!='outbox':
  1747. return
  1748. if archiveDir:
  1749. if not os.path.isdir(archiveDir):
  1750. os.mkdir(archiveDir)
  1751. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1752. postsInBox=os.listdir(boxDir)
  1753. noOfPosts=len(postsInBox)
  1754. if noOfPosts<=maxPostsInBox:
  1755. return
  1756. postsInBoxDict={}
  1757. postsCtr=0
  1758. for postFilename in postsInBox:
  1759. if not postFilename.endswith('.json'):
  1760. continue
  1761. # extract the status number
  1762. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1763. if statusNumber:
  1764. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1765. postsCtr+=1
  1766. noOfPosts=postsCtr
  1767. if noOfPosts<=maxPostsInBox:
  1768. return
  1769. # sort the list in ascending order of date
  1770. postsInBox=OrderedDict(sorted(postsInBoxDict.items(),reverse=False))
  1771. for statusNumber,postFilename in postsInBox.items():
  1772. filePath = os.path.join(boxDir, postFilename)
  1773. if os.path.isfile(filePath):
  1774. if archiveDir:
  1775. repliesPath=filePath.replace('.json','.replies')
  1776. archivePath = os.path.join(archiveDir, postFilename)
  1777. os.rename(filePath,archivePath)
  1778. if os.path.isfile(repliesPath):
  1779. os.rename(repliesPath,archivePath)
  1780. else:
  1781. deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
  1782. noOfPosts -= 1
  1783. if noOfPosts <= maxPostsInBox:
  1784. break
  1785. def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
  1786. raw: bool,simple: bool,useTor: bool, \
  1787. port: int,httpPrefix: str, \
  1788. debug: bool,projectVersion: str) -> None:
  1789. """ This is really just for test purposes
  1790. """
  1791. session = createSession(domain,port,useTor)
  1792. personCache={}
  1793. cachedWebfingers={}
  1794. federationList=[]
  1795. domainFull=domain
  1796. if port:
  1797. if port!=80 and port!=443:
  1798. if ':' not in domain:
  1799. domainFull=domain+':'+str(port)
  1800. handle=httpPrefix+"://"+domainFull+"/@"+nickname
  1801. wfRequest = \
  1802. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1803. domain,projectVersion)
  1804. if not wfRequest:
  1805. sys.exit()
  1806. personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
  1807. getPersonBox(baseDir,session,wfRequest,personCache, \
  1808. projectVersion,httpPrefix,domain,'outbox')
  1809. wfResult = json.dumps(wfRequest, indent=4, sort_keys=False)
  1810. maxMentions=10
  1811. maxEmoji=10
  1812. maxAttachments=5
  1813. userPosts = getPosts(session,personUrl,30,maxMentions,maxEmoji, \
  1814. maxAttachments,federationList, \
  1815. personCache,raw,simple,debug, \
  1816. projectVersion,httpPrefix,domain)
  1817. #print(str(userPosts))
  1818. def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
  1819. nickname: str,domain: str,port: int, \
  1820. followerUrl,updateCaps: [], \
  1821. sendThreads: [],postLog: [], \
  1822. cachedWebfingers: {},personCache: {}, \
  1823. federationList :[],debug :bool, \
  1824. projectVersion: str) -> int:
  1825. """When the capabilities for a follower are changed this
  1826. sends out an update. followerUrl is the actor of the follower.
  1827. """
  1828. updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
  1829. nickname,domain,port, \
  1830. followerUrl, \
  1831. updateCaps)
  1832. if not updateJson:
  1833. return 1
  1834. if debug:
  1835. pprint(updateJson)
  1836. print('DEBUG: sending capabilities update from '+ \
  1837. nickname+'@'+domain+' port '+ str(port) + \
  1838. ' to '+followerUrl)
  1839. clientToServer=False
  1840. followerNickname=getNicknameFromActor(followerUrl)
  1841. if not followerNickname:
  1842. print('WARN: unable to find nickname in '+followerUrl)
  1843. return 1
  1844. followerDomain,followerPort=getDomainFromActor(followerUrl)
  1845. return sendSignedJson(updateJson,session,baseDir, \
  1846. nickname,domain,port, \
  1847. followerNickname,followerDomain,followerPort, '', \
  1848. httpPrefix,True,clientToServer, \
  1849. federationList, \
  1850. sendThreads,postLog,cachedWebfingers, \
  1851. personCache,debug,projectVersion)
  1852. def populateRepliesJson(baseDir: str,nickname: str,domain: str,postRepliesFilename: str,authorized: bool,repliesJson: {}) -> None:
  1853. # populate the items list with replies
  1854. repliesBoxes=['outbox','inbox']
  1855. with open(postRepliesFilename,'r') as repliesFile:
  1856. for messageId in repliesFile:
  1857. replyFound=False
  1858. # examine inbox and outbox
  1859. for boxname in repliesBoxes:
  1860. searchFilename= \
  1861. baseDir+ \
  1862. '/accounts/'+nickname+'@'+ \
  1863. domain+'/'+ \
  1864. boxname+'/'+ \
  1865. messageId.replace('\n','').replace('/','#')+'.json'
  1866. if os.path.isfile(searchFilename):
  1867. if authorized or \
  1868. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  1869. with open(searchFilename, 'r') as fp:
  1870. postJsonObject=commentjson.load(fp)
  1871. if postJsonObject['object'].get('cc'):
  1872. if authorized or \
  1873. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  1874. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  1875. repliesJson['orderedItems'].append(postJsonObject)
  1876. replyFound=True
  1877. else:
  1878. if authorized or \
  1879. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  1880. repliesJson['orderedItems'].append(postJsonObject)
  1881. replyFound=True
  1882. break
  1883. # if not in either inbox or outbox then examine the shared inbox
  1884. if not replyFound:
  1885. searchFilename= \
  1886. baseDir+ \
  1887. '/accounts/inbox@'+ \
  1888. domain+'/inbox/'+ \
  1889. messageId.replace('\n','').replace('/','#')+'.json'
  1890. if os.path.isfile(searchFilename):
  1891. if authorized or \
  1892. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  1893. # get the json of the reply and append it to the collection
  1894. with open(searchFilename, 'r') as fp:
  1895. postJsonObject=commentjson.load(fp)
  1896. if postJsonObject['object'].get('cc'):
  1897. if authorized or \
  1898. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  1899. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  1900. repliesJson['orderedItems'].append(postJsonObject)
  1901. else:
  1902. if authorized or \
  1903. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  1904. repliesJson['orderedItems'].append(postJsonObject)