posts.py 107 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import requests
  9. import json
  10. import html
  11. import datetime
  12. import os
  13. import shutil
  14. import threading
  15. import sys
  16. import trace
  17. import time
  18. from time import gmtime, strftime
  19. from collections import OrderedDict
  20. from threads import threadWithTrace
  21. from cache import storePersonInCache
  22. from cache import getPersonFromCache
  23. from cache import expirePersonCache
  24. from pprint import pprint
  25. from random import randint
  26. from session import createSession
  27. from session import getJson
  28. from session import postJsonString
  29. from session import postImage
  30. from webfinger import webfingerHandle
  31. from httpsig import createSignedHeader
  32. from utils import removePostFromCache
  33. from utils import getCachedPostFilename
  34. from utils import getStatusNumber
  35. from utils import createPersonDir
  36. from utils import urlPermitted
  37. from utils import getNicknameFromActor
  38. from utils import getDomainFromActor
  39. from utils import deletePost
  40. from utils import validNickname
  41. from utils import locatePost
  42. from utils import loadJson
  43. from utils import saveJson
  44. from capabilities import getOcapFilename
  45. from capabilities import capabilitiesUpdate
  46. from media import attachMedia
  47. from content import addHtmlTags
  48. from content import replaceEmojiFromTags
  49. from auth import createBasicAuthHeader
  50. from config import getConfigParam
  51. from blocking import isBlocked
  52. try:
  53. from BeautifulSoup import BeautifulSoup
  54. except ImportError:
  55. from bs4 import BeautifulSoup
  56. def isModerator(baseDir: str,nickname: str) -> bool:
  57. """Returns true if the given nickname is a moderator
  58. """
  59. moderatorsFile=baseDir+'/accounts/moderators.txt'
  60. if not os.path.isfile(moderatorsFile):
  61. if getConfigParam(baseDir,'admin')==nickname:
  62. return True
  63. return False
  64. with open(moderatorsFile, "r") as f:
  65. lines = f.readlines()
  66. if len(lines)==0:
  67. if getConfigParam(baseDir,'admin')==nickname:
  68. return True
  69. for moderator in lines:
  70. moderator=moderator.strip('\n')
  71. if moderator==nickname:
  72. return True
  73. return False
  74. def noOfFollowersOnDomain(baseDir: str,handle: str, \
  75. domain: str, followFile='followers.txt') -> int:
  76. """Returns the number of followers of the given handle from the given domain
  77. """
  78. filename=baseDir+'/accounts/'+handle+'/'+followFile
  79. if not os.path.isfile(filename):
  80. return 0
  81. ctr=0
  82. with open(filename, "r") as followersFilename:
  83. for followerHandle in followersFilename:
  84. if '@' in followerHandle:
  85. followerDomain= \
  86. followerHandle.split('@')[1].replace('\n','')
  87. if domain==followerDomain:
  88. ctr+=1
  89. return ctr
  90. def getPersonKey(nickname: str,domain: str,baseDir: str,keyType='public', \
  91. debug=False):
  92. """Returns the public or private key of a person
  93. """
  94. handle=nickname+'@'+domain
  95. keyFilename=baseDir+'/keys/'+keyType+'/'+handle.lower()+'.key'
  96. if not os.path.isfile(keyFilename):
  97. if debug:
  98. print('DEBUG: private key file not found: '+keyFilename)
  99. return ''
  100. keyPem=''
  101. with open(keyFilename, "r") as pemFile:
  102. keyPem=pemFile.read()
  103. if len(keyPem)<20:
  104. if debug:
  105. print('DEBUG: private key was too short: '+keyPem)
  106. return ''
  107. return keyPem
  108. def cleanHtml(rawHtml: str) -> str:
  109. text = BeautifulSoup(rawHtml, 'html.parser').get_text()
  110. return html.unescape(text)
  111. def getUserUrl(wfRequest: {}) -> str:
  112. if wfRequest.get('links'):
  113. for link in wfRequest['links']:
  114. if link.get('type') and link.get('href'):
  115. if link['type'] == 'application/activity+json':
  116. if not ('/users/' in link['href'] or \
  117. '/profile/' in link['href'] or \
  118. '/channel/' in link['href']):
  119. print('Webfinger activity+json contains single user instance actor')
  120. return link['href']
  121. return None
  122. def parseUserFeed(session,feedUrl: str,asHeader: {}, \
  123. projectVersion: str,httpPrefix: str,domain: str) -> None:
  124. feedJson = getJson(session,feedUrl,asHeader,None, \
  125. projectVersion,httpPrefix,domain)
  126. if not feedJson:
  127. return
  128. if 'orderedItems' in feedJson:
  129. for item in feedJson['orderedItems']:
  130. yield item
  131. nextUrl = None
  132. if 'first' in feedJson:
  133. nextUrl = feedJson['first']
  134. elif 'next' in feedJson:
  135. nextUrl = feedJson['next']
  136. if nextUrl:
  137. if isinstance(nextUrl, str):
  138. userFeed=parseUserFeed(session,nextUrl,asHeader, \
  139. projectVersion,httpPrefix,domain)
  140. for item in userFeed:
  141. yield item
  142. elif isinstance(nextUrl, dict):
  143. userFeed=nextUrl
  144. if userFeed.get('orderedItems'):
  145. for item in userFeed['orderedItems']:
  146. yield item
  147. def getPersonBox(baseDir: str,session,wfRequest: {},personCache: {}, \
  148. projectVersion: str,httpPrefix: str, \
  149. nickname: str,domain: str, \
  150. boxName='inbox') -> (str,str,str,str,str,str,str,str):
  151. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  152. if not wfRequest.get('errors'):
  153. personUrl = getUserUrl(wfRequest)
  154. else:
  155. if nickname=='dev':
  156. # try single user instance
  157. print('getPersonBox: Trying single user instance with ld+json')
  158. personUrl = httpPrefix+'://'+domain
  159. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  160. else:
  161. personUrl = httpPrefix+'://'+domain+'/users/'+nickname
  162. if not personUrl:
  163. return None,None,None,None,None,None,None,None
  164. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  165. if not personJson:
  166. if '/channel/' in personUrl:
  167. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  168. personJson = getJson(session,personUrl,asHeader,None, \
  169. projectVersion,httpPrefix,domain)
  170. if not personJson:
  171. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  172. personJson = getJson(session,personUrl,asHeader,None, \
  173. projectVersion,httpPrefix,domain)
  174. if not personJson:
  175. print('Unable to get actor')
  176. return None,None,None,None,None,None,None,None
  177. boxJson=None
  178. if not personJson.get(boxName):
  179. if personJson.get('endpoints'):
  180. if personJson['endpoints'].get(boxName):
  181. boxJson=personJson['endpoints'][boxName]
  182. else:
  183. boxJson=personJson[boxName]
  184. if not boxJson:
  185. return None,None,None,None,None,None,None,None
  186. personId=None
  187. if personJson.get('id'):
  188. personId=personJson['id']
  189. pubKeyId=None
  190. pubKey=None
  191. if personJson.get('publicKey'):
  192. if personJson['publicKey'].get('id'):
  193. pubKeyId=personJson['publicKey']['id']
  194. if personJson['publicKey'].get('publicKeyPem'):
  195. pubKey=personJson['publicKey']['publicKeyPem']
  196. sharedInbox=None
  197. if personJson.get('sharedInbox'):
  198. sharedInbox=personJson['sharedInbox']
  199. else:
  200. if personJson.get('endpoints'):
  201. if personJson['endpoints'].get('sharedInbox'):
  202. sharedInbox=personJson['endpoints']['sharedInbox']
  203. capabilityAcquisition=None
  204. if personJson.get('capabilityAcquisitionEndpoint'):
  205. capabilityAcquisition=personJson['capabilityAcquisitionEndpoint']
  206. avatarUrl=None
  207. if personJson.get('icon'):
  208. if personJson['icon'].get('url'):
  209. avatarUrl=personJson['icon']['url']
  210. displayName=None
  211. if personJson.get('name'):
  212. displayName=personJson['name']
  213. storePersonInCache(baseDir,personUrl,personJson,personCache)
  214. return boxJson,pubKeyId,pubKey,personId,sharedInbox,capabilityAcquisition,avatarUrl,displayName
  215. def getPosts(session,outboxUrl: str,maxPosts: int, \
  216. maxMentions: int, \
  217. maxEmoji: int,maxAttachments: int, \
  218. federationList: [], \
  219. personCache: {},raw: bool, \
  220. simple: bool,debug: bool, \
  221. projectVersion: str,httpPrefix: str,domain: str) -> {}:
  222. """Gets public posts from an outbox
  223. """
  224. personPosts={}
  225. if not outboxUrl:
  226. return personPosts
  227. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  228. if '/outbox/' in outboxUrl:
  229. asHeader = {'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  230. if raw:
  231. result = []
  232. i = 0
  233. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  234. projectVersion,httpPrefix,domain)
  235. for item in userFeed:
  236. result.append(item)
  237. i += 1
  238. if i == maxPosts:
  239. break
  240. pprint(result)
  241. return None
  242. i = 0
  243. userFeed=parseUserFeed(session,outboxUrl,asHeader, \
  244. projectVersion,httpPrefix,domain)
  245. for item in userFeed:
  246. if not item.get('id'):
  247. if debug:
  248. print('No id')
  249. continue
  250. if not item.get('type'):
  251. if debug:
  252. print('No type')
  253. continue
  254. if item['type'] != 'Create':
  255. if debug:
  256. print('Not Create type')
  257. continue
  258. if not item.get('object'):
  259. if debug:
  260. print('No object')
  261. continue
  262. if not isinstance(item['object'], dict):
  263. if debug:
  264. print('item object is not a dict')
  265. continue
  266. if not item['object'].get('published'):
  267. if debug:
  268. print('No published attribute')
  269. continue
  270. #pprint(item)
  271. published = item['object']['published']
  272. if not personPosts.get(item['id']):
  273. # check that this is a public post
  274. # #Public should appear in the "to" list
  275. if item['object'].get('to'):
  276. isPublic=False
  277. for recipient in item['object']['to']:
  278. if recipient.endswith('#Public'):
  279. isPublic=True
  280. break
  281. if not isPublic:
  282. continue
  283. content = item['object']['content'].replace('&apos;',"'")
  284. mentions=[]
  285. emoji={}
  286. if item['object'].get('tag'):
  287. for tagItem in item['object']['tag']:
  288. tagType=tagItem['type'].lower()
  289. if tagType=='emoji':
  290. if tagItem.get('name') and tagItem.get('icon'):
  291. if tagItem['icon'].get('url'):
  292. # No emoji from non-permitted domains
  293. if urlPermitted(tagItem['icon']['url'], \
  294. federationList, \
  295. "objects:read"):
  296. emojiName=tagItem['name']
  297. emojiIcon=tagItem['icon']['url']
  298. emoji[emojiName]=emojiIcon
  299. else:
  300. if debug:
  301. print('url not permitted '+tagItem['icon']['url'])
  302. if tagType=='mention':
  303. if tagItem.get('name'):
  304. if tagItem['name'] not in mentions:
  305. mentions.append(tagItem['name'])
  306. if len(mentions)>maxMentions:
  307. if debug:
  308. print('max mentions reached')
  309. continue
  310. if len(emoji)>maxEmoji:
  311. if debug:
  312. print('max emojis reached')
  313. continue
  314. summary = ''
  315. if item['object'].get('summary'):
  316. if item['object']['summary']:
  317. summary = item['object']['summary']
  318. inReplyTo = ''
  319. if item['object'].get('inReplyTo'):
  320. if item['object']['inReplyTo']:
  321. # No replies to non-permitted domains
  322. if not urlPermitted(item['object']['inReplyTo'], \
  323. federationList, \
  324. "objects:read"):
  325. if debug:
  326. print('url not permitted '+item['object']['inReplyTo'])
  327. continue
  328. inReplyTo = item['object']['inReplyTo']
  329. conversation = ''
  330. if item['object'].get('conversation'):
  331. if item['object']['conversation']:
  332. # no conversations originated in non-permitted domains
  333. if urlPermitted(item['object']['conversation'], \
  334. federationList,"objects:read"):
  335. conversation = item['object']['conversation']
  336. attachment = []
  337. if item['object'].get('attachment'):
  338. if item['object']['attachment']:
  339. for attach in item['object']['attachment']:
  340. if attach.get('name') and attach.get('url'):
  341. # no attachments from non-permitted domains
  342. if urlPermitted(attach['url'], \
  343. federationList, \
  344. "objects:read"):
  345. attachment.append([attach['name'],attach['url']])
  346. else:
  347. if debug:
  348. print('url not permitted '+attach['url'])
  349. sensitive = False
  350. if item['object'].get('sensitive'):
  351. sensitive = item['object']['sensitive']
  352. if simple:
  353. print(cleanHtml(content)+'\n')
  354. else:
  355. pprint(item)
  356. personPosts[item['id']] = {
  357. "sensitive": sensitive,
  358. "inreplyto": inReplyTo,
  359. "summary": summary,
  360. "html": content,
  361. "plaintext": cleanHtml(content),
  362. "attachment": attachment,
  363. "mentions": mentions,
  364. "emoji": emoji,
  365. "conversation": conversation
  366. }
  367. i += 1
  368. if i == maxPosts:
  369. break
  370. return personPosts
  371. def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
  372. """Deletes all posts for a person from inbox or outbox
  373. """
  374. if boxname!='inbox' and boxname!='outbox':
  375. return
  376. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  377. for deleteFilename in os.scandir(boxDir):
  378. deleteFilename=deleteFilename.name
  379. filePath = os.path.join(boxDir, deleteFilename)
  380. try:
  381. if os.path.isfile(filePath):
  382. os.unlink(filePath)
  383. elif os.path.isdir(filePath): shutil.rmtree(filePath)
  384. except Exception as e:
  385. print(e)
  386. def savePostToBox(baseDir: str,httpPrefix: str,postId: str, \
  387. nickname: str, domain: str,postJsonObject: {}, \
  388. boxname: str) -> str:
  389. """Saves the give json to the give box
  390. Returns the filename
  391. """
  392. if boxname!='inbox' and boxname!='outbox':
  393. return None
  394. originalDomain=domain
  395. if ':' in domain:
  396. domain=domain.split(':')[0]
  397. if not postId:
  398. statusNumber,published = getStatusNumber()
  399. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  400. postJsonObject['id']=postId+'/activity'
  401. if postJsonObject.get('object'):
  402. if isinstance(postJsonObject['object'], dict):
  403. postJsonObject['object']['id']=postId
  404. postJsonObject['object']['atomUri']=postId
  405. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  406. filename=boxDir+'/'+postId.replace('/','#')+'.json'
  407. saveJson(postJsonObject,filename)
  408. return filename
  409. def updateHashtagsIndex(baseDir: str,tag: {},newPostId: str) -> None:
  410. """Writes the post url for hashtags to a file
  411. This allows posts for a hashtag to be quickly looked up
  412. """
  413. if tag['type']!='Hashtag':
  414. return
  415. # create hashtags directory
  416. tagsDir=baseDir+'/tags'
  417. if not os.path.isdir(tagsDir):
  418. os.mkdir(tagsDir)
  419. tagName=tag['name']
  420. tagsFilename=tagsDir+'/'+tagName[1:]+'.txt'
  421. tagFile=open(tagsFilename, "a+")
  422. if not tagFile:
  423. return
  424. tagFile.write(newPostId+'\n')
  425. tagFile.close()
  426. def createPostBase(baseDir: str,nickname: str,domain: str,port: int, \
  427. toUrl: str,ccUrl: str,httpPrefix: str,content: str, \
  428. followersOnly: bool,saveToFile: bool,clientToServer: bool, \
  429. attachImageFilename: str, \
  430. mediaType: str,imageDescription: str, \
  431. useBlurhash: bool,isModerationReport: bool,inReplyTo=None, \
  432. inReplyToAtomUri=None,subject=None, \
  433. eventDate=None,eventTime=None,location=None) -> {}:
  434. """Creates a message
  435. """
  436. mentionedRecipients= \
  437. getMentionedPeople(baseDir,httpPrefix,content,domain,False)
  438. tags=[]
  439. hashtagsDict={}
  440. if port:
  441. if port!=80 and port!=443:
  442. if ':' not in domain:
  443. domain=domain+':'+str(port)
  444. # convert content to html
  445. emojisDict={}
  446. # replace 's
  447. content=content.replace("\ufffd\ufffd\ufffds","'s").replace("’","'")
  448. # add tags
  449. content= \
  450. addHtmlTags(baseDir,httpPrefix, \
  451. nickname,domain,content, \
  452. mentionedRecipients, \
  453. hashtagsDict,True)
  454. statusNumber,published = getStatusNumber()
  455. postTo='https://www.w3.org/ns/activitystreams#Public'
  456. postCC=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  457. if followersOnly:
  458. postTo=postCC
  459. postCC=''
  460. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  461. sensitive=False
  462. summary=None
  463. if subject:
  464. summary=subject
  465. sensitive=True
  466. toRecipients=[]
  467. toCC=[]
  468. if toUrl:
  469. if not isinstance(toUrl, str):
  470. print('ERROR: toUrl is not a string')
  471. return None
  472. toRecipients=[toUrl]
  473. # who to send to
  474. if mentionedRecipients:
  475. for mention in mentionedRecipients:
  476. if mention not in toCC:
  477. toCC.append(mention)
  478. # create a list of hashtags
  479. # Only posts which are #Public are searchable by hashtag
  480. if hashtagsDict:
  481. isPublic=False
  482. for recipient in toRecipients:
  483. if recipient.endswith('#Public'):
  484. isPublic=True
  485. break
  486. for tagName,tag in hashtagsDict.items():
  487. tags.append(tag)
  488. if isPublic:
  489. updateHashtagsIndex(baseDir,tag,newPostId)
  490. print('Content tags: '+str(tags))
  491. if inReplyTo and not sensitive:
  492. # locate the post which this is a reply to and check if
  493. # it has a content warning. If it does then reproduce
  494. # the same warning
  495. replyPostFilename=locatePost(baseDir,nickname,domain,inReplyTo)
  496. if replyPostFilename:
  497. replyToJson=loadJson(replyPostFilename)
  498. if replyToJson:
  499. if replyToJson.get('object'):
  500. if replyToJson['object'].get('sensitive'):
  501. if replyToJson['object']['sensitive']:
  502. sensitive=True
  503. if replyToJson['object'].get('summary'):
  504. summary=replyToJson['object']['summary']
  505. if eventDate:
  506. eventName=summary
  507. if not eventName:
  508. eventName=content
  509. eventDateStr=eventDate
  510. if eventTime:
  511. if eventTime.endswith('Z'):
  512. eventDateStr=eventDate+'T'+eventTime
  513. else:
  514. eventDateStr=eventDate+'T'+eventTime+':00'+strftime("%z", gmtime())
  515. else:
  516. eventDateStr=eventDate+'T12:00:00Z'
  517. tags.append({
  518. "@context": "https://www.w3.org/ns/activitystreams",
  519. "type": "Event",
  520. "name": eventName,
  521. "startTime": eventDateStr,
  522. "endTime": eventDateStr
  523. })
  524. if location:
  525. tags.append({
  526. "@context": "https://www.w3.org/ns/activitystreams",
  527. "type": "Place",
  528. "name": location
  529. })
  530. postContext=[
  531. 'https://www.w3.org/ns/activitystreams',
  532. {
  533. 'Hashtag': 'as:Hashtag',
  534. 'sensitive': 'as:sensitive',
  535. 'toot': 'http://joinmastodon.org/ns#',
  536. 'votersCount': 'toot:votersCount'
  537. }
  538. ]
  539. if not clientToServer:
  540. actorUrl=httpPrefix+'://'+domain+'/users/'+nickname
  541. # if capabilities have been granted for this actor
  542. # then get the corresponding id
  543. capabilityId=None
  544. capabilityIdList=[]
  545. ocapFilename=getOcapFilename(baseDir,nickname,domain,toUrl,'granted')
  546. if ocapFilename:
  547. if os.path.isfile(ocapFilename):
  548. oc=loadJson(ocapFilename)
  549. if oc:
  550. if oc.get('id'):
  551. capabilityIdList=[oc['id']]
  552. newPost = {
  553. "@context": postContext,
  554. 'id': newPostId+'/activity',
  555. 'capability': capabilityIdList,
  556. 'type': 'Create',
  557. 'actor': actorUrl,
  558. 'published': published,
  559. 'to': toRecipients,
  560. 'cc': toCC,
  561. 'object': {
  562. 'id': newPostId,
  563. 'type': 'Note',
  564. 'summary': summary,
  565. 'inReplyTo': inReplyTo,
  566. 'published': published,
  567. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  568. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  569. 'to': toRecipients,
  570. 'cc': toCC,
  571. 'sensitive': sensitive,
  572. 'atomUri': newPostId,
  573. 'inReplyToAtomUri': inReplyToAtomUri,
  574. 'content': content,
  575. 'contentMap': {
  576. 'en': content
  577. },
  578. 'attachment': [],
  579. 'tag': tags,
  580. 'replies': {
  581. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  582. 'type': 'Collection',
  583. 'first': {
  584. 'type': 'CollectionPage',
  585. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  586. 'items': []
  587. }
  588. }
  589. }
  590. }
  591. if attachImageFilename:
  592. newPost['object']= \
  593. attachMedia(baseDir,httpPrefix,domain,port, \
  594. newPost['object'],attachImageFilename, \
  595. mediaType,imageDescription,useBlurhash)
  596. else:
  597. newPost = {
  598. "@context": postContext,
  599. 'id': newPostId,
  600. 'type': 'Note',
  601. 'summary': summary,
  602. 'inReplyTo': inReplyTo,
  603. 'published': published,
  604. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  605. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  606. 'to': toRecipients,
  607. 'cc': toCC,
  608. 'sensitive': sensitive,
  609. 'atomUri': newPostId,
  610. 'inReplyToAtomUri': inReplyToAtomUri,
  611. 'content': content,
  612. 'contentMap': {
  613. 'en': content
  614. },
  615. 'attachment': [],
  616. 'tag': tags,
  617. 'replies': {
  618. 'id': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  619. 'type': 'Collection',
  620. 'first': {
  621. 'type': 'CollectionPage',
  622. 'partOf': 'https://'+domain+'/users/'+nickname+'/statuses/'+statusNumber+'/replies',
  623. 'items': []
  624. }
  625. }
  626. }
  627. if attachImageFilename:
  628. newPost= \
  629. attachMedia(baseDir,httpPrefix,domain,port, \
  630. newPost,attachImageFilename, \
  631. mediaType,imageDescription,useBlurhash)
  632. if ccUrl:
  633. if len(ccUrl)>0:
  634. newPost['cc']=[ccUrl]
  635. if newPost.get('object'):
  636. newPost['object']['cc']=[ccUrl]
  637. # if this is a moderation report then add a status
  638. if isModerationReport:
  639. # add status
  640. if newPost.get('object'):
  641. newPost['object']['moderationStatus']='pending'
  642. else:
  643. newPost['moderationStatus']='pending'
  644. # save to index file
  645. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  646. modFile=open(moderationIndexFile, "a+")
  647. if modFile:
  648. modFile.write(newPostId+'\n')
  649. modFile.close()
  650. if saveToFile:
  651. savePostToBox(baseDir,httpPrefix,newPostId, \
  652. nickname,domain,newPost,'outbox')
  653. return newPost
  654. def outboxMessageCreateWrap(httpPrefix: str, \
  655. nickname: str,domain: str,port: int, \
  656. messageJson: {}) -> {}:
  657. """Wraps a received message in a Create
  658. https://www.w3.org/TR/activitypub/#object-without-create
  659. """
  660. if port:
  661. if port!=80 and port!=443:
  662. if ':' not in domain:
  663. domain=domain+':'+str(port)
  664. statusNumber,published = getStatusNumber()
  665. if messageJson.get('published'):
  666. published = messageJson['published']
  667. newPostId=httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  668. cc=[]
  669. if messageJson.get('cc'):
  670. cc=messageJson['cc']
  671. # TODO
  672. capabilityUrl=[]
  673. newPost = {
  674. "@context": "https://www.w3.org/ns/activitystreams",
  675. 'id': newPostId+'/activity',
  676. 'capability': capabilityUrl,
  677. 'type': 'Create',
  678. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  679. 'published': published,
  680. 'to': messageJson['to'],
  681. 'cc': cc,
  682. 'object': messageJson
  683. }
  684. newPost['object']['id']=newPost['id']
  685. newPost['object']['url']= \
  686. httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber
  687. newPost['object']['atomUri']= \
  688. httpPrefix+'://'+domain+'/users/'+nickname+'/statuses/'+statusNumber
  689. return newPost
  690. def postIsAddressedToFollowers(baseDir: str,
  691. nickname: str,domain: str,port: int, \
  692. httpPrefix: str,
  693. postJsonObject: {}) -> bool:
  694. """Returns true if the given post is addressed to followers of the nickname
  695. """
  696. if port:
  697. if port!=80 and port!=443:
  698. if ':' not in domain:
  699. domain=domain+':'+str(port)
  700. if not postJsonObject.get('object'):
  701. return False
  702. toList=[]
  703. ccList=[]
  704. if postJsonObject['type']!='Update' and \
  705. isinstance(postJsonObject['object'], dict):
  706. if postJsonObject['object'].get('to'):
  707. toList=postJsonObject['object']['to']
  708. if postJsonObject['object'].get('cc'):
  709. ccList=postJsonObject['object']['cc']
  710. else:
  711. if postJsonObject.get('to'):
  712. toList=postJsonObject['to']
  713. if postJsonObject.get('cc'):
  714. ccList=postJsonObject['cc']
  715. followersUrl=httpPrefix+'://'+domain+'/users/'+nickname+'/followers'
  716. # does the followers url exist in 'to' or 'cc' lists?
  717. addressedToFollowers=False
  718. if followersUrl in toList:
  719. addressedToFollowers=True
  720. elif followersUrl in ccList:
  721. addressedToFollowers=True
  722. return addressedToFollowers
  723. def postIsAddressedToPublic(baseDir: str,postJsonObject: {}) -> bool:
  724. """Returns true if the given post is addressed to public
  725. """
  726. if not postJsonObject.get('object'):
  727. return False
  728. if not postJsonObject['object'].get('to'):
  729. return False
  730. publicUrl='https://www.w3.org/ns/activitystreams#Public'
  731. # does the public url exist in 'to' or 'cc' lists?
  732. addressedToPublic=False
  733. if publicUrl in postJsonObject['object']['to']:
  734. addressedToPublic=True
  735. if not addressedToPublic:
  736. if not postJsonObject['object'].get('cc'):
  737. return False
  738. if publicUrl in postJsonObject['object']['cc']:
  739. addressedToPublic=True
  740. return addressedToPublic
  741. def createPublicPost(baseDir: str, \
  742. nickname: str,domain: str,port: int,httpPrefix: str, \
  743. content: str,followersOnly: bool,saveToFile: bool,
  744. clientToServer: bool,\
  745. attachImageFilename: str,mediaType: str, \
  746. imageDescription: str,useBlurhash: bool, \
  747. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  748. eventDate=None,eventTime=None,location=None) -> {}:
  749. """Public post
  750. """
  751. domainFull=domain
  752. if port:
  753. if port!=80 and port!=443:
  754. if ':' not in domain:
  755. domainFull=domain+':'+str(port)
  756. return createPostBase(baseDir,nickname,domain,port, \
  757. 'https://www.w3.org/ns/activitystreams#Public', \
  758. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  759. httpPrefix,content,followersOnly,saveToFile, \
  760. clientToServer, \
  761. attachImageFilename,mediaType, \
  762. imageDescription,useBlurhash, \
  763. False,inReplyTo,inReplyToAtomUri,subject, \
  764. eventDate,eventTime,location)
  765. def createQuestionPost(baseDir: str,
  766. nickname: str,domain: str,port: int,httpPrefix: str, \
  767. content: str,qOptions: [], \
  768. followersOnly: bool,saveToFile: bool,
  769. clientToServer: bool,\
  770. attachImageFilename: str,mediaType: str, \
  771. imageDescription: str,useBlurhash: bool, \
  772. subject: str,durationDays: int) -> {}:
  773. """Question post with multiple choice options
  774. """
  775. domainFull=domain
  776. if port:
  777. if port!=80 and port!=443:
  778. if ':' not in domain:
  779. domainFull=domain+':'+str(port)
  780. messageJson= \
  781. createPostBase(baseDir,nickname,domain,port, \
  782. 'https://www.w3.org/ns/activitystreams#Public', \
  783. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  784. httpPrefix,content,followersOnly,saveToFile, \
  785. clientToServer, \
  786. attachImageFilename,mediaType, \
  787. imageDescription,useBlurhash, \
  788. False,None,None,subject, \
  789. None,None,None)
  790. messageJson['object']['type']='Question'
  791. messageJson['object']['oneOf']=[]
  792. messageJson['object']['votersCount']=0
  793. currTime=datetime.datetime.utcnow()
  794. daysSinceEpoch=int((currTime - datetime.datetime(1970,1,1)).days + durationDays)
  795. endTime=datetime.datetime(1970,1,1) + datetime.timedelta(daysSinceEpoch)
  796. messageJson['object']['endTime']=endTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  797. for questionOption in qOptions:
  798. messageJson['object']['oneOf'].append({
  799. "type": "Note",
  800. "name": questionOption,
  801. "replies": {
  802. "type": "Collection",
  803. "totalItems": 0
  804. }
  805. })
  806. return messageJson
  807. def createUnlistedPost(baseDir: str,
  808. nickname: str,domain: str,port: int,httpPrefix: str, \
  809. content: str,followersOnly: bool,saveToFile: bool,
  810. clientToServer: bool,\
  811. attachImageFilename: str,mediaType: str, \
  812. imageDescription: str,useBlurhash: bool, \
  813. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  814. eventDate=None,eventTime=None,location=None) -> {}:
  815. """Unlisted post. This has the #Public and followers links inverted.
  816. """
  817. domainFull=domain
  818. if port:
  819. if port!=80 and port!=443:
  820. if ':' not in domain:
  821. domainFull=domain+':'+str(port)
  822. return createPostBase(baseDir,nickname,domain,port, \
  823. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  824. 'https://www.w3.org/ns/activitystreams#Public', \
  825. httpPrefix,content,followersOnly,saveToFile, \
  826. clientToServer, \
  827. attachImageFilename,mediaType, \
  828. imageDescription,useBlurhash, \
  829. False,inReplyTo, inReplyToAtomUri, subject, \
  830. eventDate,eventTime,location)
  831. def createFollowersOnlyPost(baseDir: str,
  832. nickname: str,domain: str,port: int,httpPrefix: str, \
  833. content: str,followersOnly: bool,saveToFile: bool,
  834. clientToServer: bool,\
  835. attachImageFilename: str,mediaType: str, \
  836. imageDescription: str,useBlurhash: bool, \
  837. inReplyTo=None,inReplyToAtomUri=None,subject=None, \
  838. eventDate=None,eventTime=None,location=None) -> {}:
  839. """Followers only post
  840. """
  841. domainFull=domain
  842. if port:
  843. if port!=80 and port!=443:
  844. if ':' not in domain:
  845. domainFull=domain+':'+str(port)
  846. return createPostBase(baseDir,nickname,domain,port, \
  847. httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers', \
  848. None,
  849. httpPrefix,content,followersOnly,saveToFile, \
  850. clientToServer, \
  851. attachImageFilename,mediaType, \
  852. imageDescription,useBlurhash, \
  853. False,inReplyTo, inReplyToAtomUri, subject, \
  854. eventDate,eventTime,location)
  855. def getMentionedPeople(baseDir: str,httpPrefix: str, \
  856. content: str,domain: str,debug: bool) -> []:
  857. """Extracts a list of mentioned actors from the given message content
  858. """
  859. if '@' not in content:
  860. return None
  861. mentions=[]
  862. words=content.split(' ')
  863. for wrd in words:
  864. if wrd.startswith('@'):
  865. handle=wrd[1:]
  866. if debug:
  867. print('DEBUG: mentioned handle '+handle)
  868. if '@' not in handle:
  869. handle=handle+'@'+domain
  870. if not os.path.isdir(baseDir+'/accounts/'+handle):
  871. continue
  872. else:
  873. externalDomain=handle.split('@')[1]
  874. if not ('.' in externalDomain or externalDomain=='localhost'):
  875. continue
  876. mentionedNickname=handle.split('@')[0]
  877. mentionedDomain=handle.split('@')[1].strip('\n')
  878. if ':' in mentionedDomain:
  879. mentionedDomain=mentionedDomain.split(':')[0]
  880. if not validNickname(mentionedDomain,mentionedNickname):
  881. continue
  882. actor= \
  883. httpPrefix+'://'+handle.split('@')[1]+ \
  884. '/users/'+mentionedNickname
  885. mentions.append(actor)
  886. return mentions
  887. def createDirectMessagePost(baseDir: str,
  888. nickname: str,domain: str,port: int,httpPrefix: str, \
  889. content: str,followersOnly: bool,saveToFile: bool,
  890. clientToServer: bool,\
  891. attachImageFilename: str,mediaType: str, \
  892. imageDescription: str,useBlurhash: bool, \
  893. inReplyTo=None,inReplyToAtomUri=None, \
  894. subject=None,debug=False, \
  895. eventDate=None,eventTime=None,location=None) -> {}:
  896. """Direct Message post
  897. """
  898. mentionedPeople=getMentionedPeople(baseDir,httpPrefix,content,domain,debug)
  899. if debug:
  900. print('mentionedPeople: '+str(mentionedPeople))
  901. if not mentionedPeople:
  902. return None
  903. postTo=None
  904. postCc=None
  905. messageJson= \
  906. createPostBase(baseDir,nickname,domain,port, \
  907. postTo,postCc, \
  908. httpPrefix,content,followersOnly,saveToFile, \
  909. clientToServer, \
  910. attachImageFilename,mediaType, \
  911. imageDescription,useBlurhash, \
  912. False,inReplyTo,inReplyToAtomUri,subject, \
  913. eventDate,eventTime,location)
  914. # mentioned recipients go into To rather than Cc
  915. messageJson['to']=messageJson['object']['cc']
  916. messageJson['object']['to']=messageJson['to']
  917. messageJson['cc']=[]
  918. messageJson['object']['cc']=[]
  919. return messageJson
  920. def createReportPost(baseDir: str,
  921. nickname: str, domain: str, port: int,httpPrefix: str, \
  922. content: str, followersOnly: bool, saveToFile: bool,
  923. clientToServer: bool,\
  924. attachImageFilename: str,mediaType: str, \
  925. imageDescription: str,useBlurhash: bool, \
  926. debug: bool,subject=None) -> {}:
  927. """Send a report to moderators
  928. """
  929. domainFull=domain
  930. if port:
  931. if port!=80 and port!=443:
  932. if ':' not in domain:
  933. domainFull=domain+':'+str(port)
  934. # add a title to distinguish moderation reports from other posts
  935. reportTitle='Moderation Report'
  936. if not subject:
  937. subject=reportTitle
  938. else:
  939. if not subject.startswith(reportTitle):
  940. subject=reportTitle+': '+subject
  941. # create the list of moderators from the moderators file
  942. moderatorsList=[]
  943. moderatorsFile=baseDir+'/accounts/moderators.txt'
  944. if os.path.isfile(moderatorsFile):
  945. with open (moderatorsFile, "r") as fileHandler:
  946. for line in fileHandler:
  947. line=line.strip('\n')
  948. if line.startswith('#'):
  949. continue
  950. if line.startswith('/users/'):
  951. line=line.replace('users','')
  952. if line.startswith('@'):
  953. line=line[1:]
  954. if '@' in line:
  955. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line.split('@')[0]
  956. if moderatorActor not in moderatorList:
  957. moderatorsList.append(moderatorActor)
  958. continue
  959. if line.startswith('http') or line.startswith('dat'):
  960. # must be a local address - no remote moderators
  961. if '://'+domainFull+'/' in line:
  962. if line not in moderatorsList:
  963. moderatorsList.append(line)
  964. else:
  965. if '/' not in line:
  966. moderatorActor=httpPrefix+'://'+domainFull+'/users/'+line
  967. if moderatorActor not in moderatorsList:
  968. moderatorsList.append(moderatorActor)
  969. if len(moderatorsList)==0:
  970. # if there are no moderators then the admin becomes the moderator
  971. adminNickname=getConfigParam(baseDir,'admin')
  972. if adminNickname:
  973. moderatorsList.append(httpPrefix+'://'+domainFull+ \
  974. '/users/'+adminNickname)
  975. if not moderatorsList:
  976. return None
  977. if debug:
  978. print('DEBUG: Sending report to moderators')
  979. print(str(moderatorsList))
  980. postTo=moderatorsList
  981. postCc=None
  982. postJsonObject=None
  983. for toUrl in postTo:
  984. # who is this report going to?
  985. toNickname=toUrl.split('/users/')[1]
  986. handle=toNickname+'@'+domain
  987. postJsonObject= \
  988. createPostBase(baseDir,nickname,domain,port, \
  989. toUrl,postCc, \
  990. httpPrefix,content,followersOnly,saveToFile, \
  991. clientToServer, \
  992. attachImageFilename,mediaType, \
  993. imageDescription,useBlurhash, \
  994. True,None,None,subject, \
  995. None,None,None)
  996. if not postJsonObject:
  997. continue
  998. # update the inbox index with the report filename
  999. #indexFilename=baseDir+'/accounts/'+handle+'/inbox.index'
  1000. #indexEntry=postJsonObject['id'].replace('/activity','').replace('/','#')+'.json'
  1001. #if indexEntry not in open(indexFilename).read():
  1002. # try:
  1003. # with open(indexFilename, 'a+') as fp:
  1004. # fp.write(indexEntry)
  1005. # except:
  1006. # pass
  1007. # save a notification file so that the moderator
  1008. # knows something new has appeared
  1009. newReportFile=baseDir+'/accounts/'+handle+'/.newReport'
  1010. if os.path.isfile(newReportFile):
  1011. continue
  1012. try:
  1013. with open(newReportFile, 'w') as fp:
  1014. fp.write(toUrl+'/moderation')
  1015. except:
  1016. pass
  1017. return postJsonObject
  1018. def threadSendPost(session,postJsonStr: str,federationList: [],\
  1019. inboxUrl: str, baseDir: str, \
  1020. signatureHeaderJson: {},postLog: [], \
  1021. debug :bool) -> None:
  1022. """Sends a with retries
  1023. """
  1024. tries=0
  1025. sendIntervalSec=30
  1026. for attempt in range(20):
  1027. postResult=None
  1028. unauthorized=False
  1029. try:
  1030. postResult,unauthorized = \
  1031. postJsonString(session,postJsonStr,federationList, \
  1032. inboxUrl,signatureHeaderJson, \
  1033. "inbox:write",debug)
  1034. except Exception as e:
  1035. print('ERROR: postJsonString failed '+str(e))
  1036. if unauthorized==True:
  1037. print(postJsonStr)
  1038. print('threadSendPost: Post is unauthorized')
  1039. break
  1040. if postResult:
  1041. logStr='Success on try '+str(tries)+': '+postJsonStr
  1042. else:
  1043. logStr='Retry '+str(tries)+': '+postJsonStr
  1044. postLog.append(logStr)
  1045. # keep the length of the log finite
  1046. # Don't accumulate massive files on systems with limited resources
  1047. while len(postLog)>16:
  1048. postLog.pop(0)
  1049. if debug:
  1050. # save the log file
  1051. postLogFilename=baseDir+'/post.log'
  1052. with open(postLogFilename, "a+") as logFile:
  1053. logFile.write(logStr+'\n')
  1054. if postResult:
  1055. if debug:
  1056. print('DEBUG: successful json post to '+inboxUrl)
  1057. # our work here is done
  1058. break
  1059. if debug:
  1060. print(postJsonStr)
  1061. print('DEBUG: json post to '+inboxUrl+' failed. Waiting for '+ \
  1062. str(sendIntervalSec)+' seconds.')
  1063. time.sleep(sendIntervalSec)
  1064. tries+=1
  1065. def sendPost(projectVersion: str, \
  1066. session,baseDir: str,nickname: str, domain: str, port: int, \
  1067. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1068. httpPrefix: str, content: str, followersOnly: bool, \
  1069. saveToFile: bool, clientToServer: bool, \
  1070. attachImageFilename: str,mediaType: str, \
  1071. imageDescription: str,useBlurhash: bool, \
  1072. federationList: [],\
  1073. sendThreads: [], postLog: [], cachedWebfingers: {},personCache: {}, \
  1074. debug=False,inReplyTo=None,inReplyToAtomUri=None,subject=None) -> int:
  1075. """Post to another inbox
  1076. """
  1077. withDigest=True
  1078. if toNickname=='inbox':
  1079. # shared inbox actor on @domain@domain
  1080. toNickname=toDomain
  1081. toDomainOriginal=toDomain
  1082. if toPort:
  1083. if toPort!=80 and toPort!=443:
  1084. if ':' not in toDomain:
  1085. toDomain=toDomain+':'+str(toPort)
  1086. handle=httpPrefix+'://'+toDomain+'/@'+toNickname
  1087. # lookup the inbox for the To handle
  1088. wfRequest = webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1089. domain,projectVersion)
  1090. if not wfRequest:
  1091. return 1
  1092. if not clientToServer:
  1093. postToBox='inbox'
  1094. else:
  1095. postToBox='outbox'
  1096. # get the actor inbox for the To handle
  1097. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1098. getPersonBox(baseDir,session,wfRequest,personCache, \
  1099. projectVersion,httpPrefix, \
  1100. nickname,domain,postToBox)
  1101. # If there are more than one followers on the target domain
  1102. # then send to the shared inbox indead of the individual inbox
  1103. if nickname=='capabilities':
  1104. inboxUrl=capabilityAcquisition
  1105. if not capabilityAcquisition:
  1106. return 2
  1107. if not inboxUrl:
  1108. return 3
  1109. if not pubKey:
  1110. return 4
  1111. if not toPersonId:
  1112. return 5
  1113. # sharedInbox and capabilities are optional
  1114. postJsonObject = \
  1115. createPostBase(baseDir,nickname,domain,port, \
  1116. toPersonId,cc,httpPrefix,content, \
  1117. followersOnly,saveToFile,clientToServer, \
  1118. attachImageFilename,mediaType, \
  1119. imageDescription,useBlurhash, \
  1120. False,inReplyTo,inReplyToAtomUri,subject, \
  1121. None,None,None)
  1122. # get the senders private key
  1123. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private')
  1124. if len(privateKeyPem)==0:
  1125. return 6
  1126. if toDomain not in inboxUrl:
  1127. return 7
  1128. postPath=inboxUrl.split(toDomain,1)[1]
  1129. # convert json to string so that there are no
  1130. # subsequent conversions after creating message body digest
  1131. postJsonStr=json.dumps(postJsonObject)
  1132. # construct the http header, including the message body digest
  1133. signatureHeaderJson = \
  1134. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1135. toDomain,toPort, \
  1136. postPath,httpPrefix,withDigest,postJsonStr)
  1137. # Keep the number of threads being used small
  1138. while len(sendThreads)>1000:
  1139. print('WARN: Maximum threads reached - killing send thread')
  1140. sendThreads[0].kill()
  1141. sendThreads.pop(0)
  1142. print('WARN: thread killed')
  1143. thr = \
  1144. threadWithTrace(target=threadSendPost, \
  1145. args=(session, \
  1146. postJsonStr, \
  1147. federationList, \
  1148. inboxUrl,baseDir, \
  1149. signatureHeaderJson.copy(), \
  1150. postLog,
  1151. debug),daemon=True)
  1152. sendThreads.append(thr)
  1153. thr.start()
  1154. return 0
  1155. def sendPostViaServer(projectVersion: str, \
  1156. baseDir: str,session,fromNickname: str,password: str, \
  1157. fromDomain: str, fromPort: int, \
  1158. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1159. httpPrefix: str, content: str, followersOnly: bool, \
  1160. attachImageFilename: str,mediaType: str, \
  1161. imageDescription: str,useBlurhash: bool, \
  1162. cachedWebfingers: {},personCache: {}, \
  1163. debug=False,inReplyTo=None, \
  1164. inReplyToAtomUri=None,subject=None) -> int:
  1165. """Send a post via a proxy (c2s)
  1166. """
  1167. if not session:
  1168. print('WARN: No session for sendPostViaServer')
  1169. return 6
  1170. withDigest=True
  1171. if toPort:
  1172. if toPort!=80 and toPort!=443:
  1173. if ':' not in fromDomain:
  1174. fromDomain=fromDomain+':'+str(fromPort)
  1175. handle=httpPrefix+'://'+fromDomain+'/@'+fromNickname
  1176. # lookup the inbox for the To handle
  1177. wfRequest = \
  1178. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1179. fromDomain,projectVersion)
  1180. if not wfRequest:
  1181. if debug:
  1182. print('DEBUG: webfinger failed for '+handle)
  1183. return 1
  1184. postToBox='outbox'
  1185. # get the actor inbox for the To handle
  1186. inboxUrl,pubKeyId,pubKey,fromPersonId,sharedInbox,capabilityAcquisition,avatarUrl,displayName = \
  1187. getPersonBox(baseDir,session,wfRequest,personCache, \
  1188. projectVersion,httpPrefix,fromNickname, \
  1189. fromDomain,postToBox)
  1190. if not inboxUrl:
  1191. if debug:
  1192. print('DEBUG: No '+postToBox+' was found for '+handle)
  1193. return 3
  1194. if not fromPersonId:
  1195. if debug:
  1196. print('DEBUG: No actor was found for '+handle)
  1197. return 4
  1198. # Get the json for the c2s post, not saving anything to file
  1199. # Note that baseDir is set to None
  1200. saveToFile=False
  1201. clientToServer=True
  1202. if toDomain.lower().endswith('public'):
  1203. toPersonId='https://www.w3.org/ns/activitystreams#Public'
  1204. fromDomainFull=fromDomain
  1205. if fromPort:
  1206. if fromPort!=80 and fromPort!=443:
  1207. if ':' not in fromDomain:
  1208. fromDomainFull=fromDomain+':'+str(fromPort)
  1209. cc=httpPrefix+'://'+fromDomainFull+'/users/'+fromNickname+'/followers'
  1210. else:
  1211. if toDomain.lower().endswith('followers') or \
  1212. toDomain.lower().endswith('followersonly'):
  1213. toPersonId= \
  1214. httpPrefix+'://'+ \
  1215. fromDomainFull+'/users/'+fromNickname+'/followers'
  1216. else:
  1217. toDomainFull=toDomain
  1218. if toPort:
  1219. if toPort!=80 and toPort!=443:
  1220. if ':' not in toDomain:
  1221. toDomainFull=toDomain+':'+str(toPort)
  1222. toPersonId=httpPrefix+'://'+toDomainFull+'/users/'+toNickname
  1223. postJsonObject = \
  1224. createPostBase(baseDir, \
  1225. fromNickname,fromDomain,fromPort, \
  1226. toPersonId,cc,httpPrefix,content, \
  1227. followersOnly,saveToFile,clientToServer, \
  1228. attachImageFilename,mediaType, \
  1229. imageDescription,useBlurhash, \
  1230. False,inReplyTo,inReplyToAtomUri,subject, \
  1231. None,None,None)
  1232. authHeader=createBasicAuthHeader(fromNickname,password)
  1233. if attachImageFilename:
  1234. headers = {'host': fromDomain, \
  1235. 'Authorization': authHeader}
  1236. postResult = \
  1237. postImage(session,attachImageFilename,[], \
  1238. inboxUrl,headers,"inbox:write")
  1239. #if not postResult:
  1240. # if debug:
  1241. # print('DEBUG: Failed to upload image')
  1242. # return 9
  1243. headers = {'host': fromDomain, \
  1244. 'Content-type': 'application/json', \
  1245. 'Authorization': authHeader}
  1246. postResult = \
  1247. postJsonString(session,json.dumps(postJsonObject),[], \
  1248. inboxUrl,headers,"inbox:write",debug)
  1249. #if not postResult:
  1250. # if debug:
  1251. # print('DEBUG: POST failed for c2s to '+inboxUrl)
  1252. # return 5
  1253. if debug:
  1254. print('DEBUG: c2s POST success')
  1255. return 0
  1256. def groupFollowersByDomain(baseDir :str,nickname :str,domain :str) -> {}:
  1257. """Returns a dictionary with followers grouped by domain
  1258. """
  1259. handle=nickname+'@'+domain
  1260. followersFilename=baseDir+'/accounts/'+handle+'/followers.txt'
  1261. if not os.path.isfile(followersFilename):
  1262. return None
  1263. grouped={}
  1264. with open(followersFilename, "r") as f:
  1265. for followerHandle in f:
  1266. if '@' in followerHandle:
  1267. fHandle=followerHandle.strip().replace('\n','')
  1268. followerDomain=fHandle.split('@')[1]
  1269. if not grouped.get(followerDomain):
  1270. grouped[followerDomain]=[fHandle]
  1271. else:
  1272. grouped[followerDomain].append(fHandle)
  1273. return grouped
  1274. def addFollowersToPublicPost(postJsonObject: {}) -> None:
  1275. """Adds followers entry to cc if it doesn't exist
  1276. """
  1277. if not postJsonObject.get('actor'):
  1278. return
  1279. if isinstance(postJsonObject['object'], str):
  1280. if not postJsonObject.get('to'):
  1281. return
  1282. if len(postJsonObject['to'])>1:
  1283. return
  1284. if len(postJsonObject['to'])==0:
  1285. return
  1286. if not postJsonObject['to'][0].endswith('#Public'):
  1287. return
  1288. if postJsonObject.get('cc'):
  1289. return
  1290. postJsonObject['cc']=postJsonObject['actor']+'/followers'
  1291. elif isinstance(postJsonObject['object'], dict):
  1292. if not postJsonObject['object'].get('to'):
  1293. return
  1294. if len(postJsonObject['object']['to'])>1:
  1295. return
  1296. if len(postJsonObject['object']['to'])==0:
  1297. return
  1298. if not postJsonObject['object']['to'][0].endswith('#Public'):
  1299. return
  1300. if postJsonObject['object'].get('cc'):
  1301. return
  1302. postJsonObject['object']['cc']=postJsonObject['actor']+'/followers'
  1303. def sendSignedJson(postJsonObject: {},session,baseDir: str, \
  1304. nickname: str, domain: str, port: int, \
  1305. toNickname: str, toDomain: str, toPort: int, cc: str, \
  1306. httpPrefix: str, saveToFile: bool, clientToServer: bool, \
  1307. federationList: [], \
  1308. sendThreads: [], postLog: [], cachedWebfingers: {}, \
  1309. personCache: {}, debug: bool,projectVersion: str) -> int:
  1310. """Sends a signed json object to an inbox/outbox
  1311. """
  1312. if debug:
  1313. print('DEBUG: sendSignedJson start')
  1314. if not session:
  1315. print('WARN: No session specified for sendSignedJson')
  1316. return 8
  1317. withDigest=True
  1318. sharedInbox=False
  1319. if toNickname=='inbox':
  1320. # shared inbox actor on @domain@domain
  1321. toNickname=toDomain
  1322. sharedInbox=True
  1323. toDomainOriginal=toDomain
  1324. if toPort:
  1325. if toPort!=80 and toPort!=443:
  1326. if ':' not in toDomain:
  1327. toDomain=toDomain+':'+str(toPort)
  1328. handleBase=httpPrefix+'://'+toDomain+'/@'
  1329. if toNickname:
  1330. handle=handleBase+toNickname
  1331. else:
  1332. singleUserInstanceNickname='dev'
  1333. handle=handleBase+singleUserInstanceNickname
  1334. if debug:
  1335. print('DEBUG: handle - '+handle+' toPort '+str(toPort))
  1336. # lookup the inbox for the To handle
  1337. wfRequest=webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  1338. domain,projectVersion)
  1339. if not wfRequest:
  1340. if debug:
  1341. print('DEBUG: webfinger for '+handle+' failed')
  1342. return 1
  1343. if wfRequest.get('errors'):
  1344. if debug:
  1345. print('DEBUG: webfinger for '+handle+' failed with errors '+str(wfRequest['errors']))
  1346. if not clientToServer:
  1347. postToBox='inbox'
  1348. else:
  1349. postToBox='outbox'
  1350. # get the actor inbox/outbox/capabilities for the To handle
  1351. inboxUrl,pubKeyId,pubKey,toPersonId,sharedInboxUrl,capabilityAcquisition,avatarUrl,displayName = \
  1352. getPersonBox(baseDir,session,wfRequest,personCache, \
  1353. projectVersion,httpPrefix,nickname,domain,postToBox)
  1354. if nickname=='capabilities':
  1355. inboxUrl=capabilityAcquisition
  1356. if not capabilityAcquisition:
  1357. return 2
  1358. else:
  1359. print("inboxUrl: "+str(inboxUrl))
  1360. print("toPersonId: "+str(toPersonId))
  1361. print("sharedInboxUrl: "+str(sharedInboxUrl))
  1362. if inboxUrl:
  1363. if inboxUrl.endswith('/actor/inbox'):
  1364. inboxUrl=sharedInboxUrl
  1365. if not inboxUrl:
  1366. if debug:
  1367. print('DEBUG: missing inboxUrl')
  1368. return 3
  1369. if debug:
  1370. print('DEBUG: Sending to endpoint '+inboxUrl)
  1371. if not pubKey:
  1372. if debug:
  1373. print('DEBUG: missing pubkey')
  1374. return 4
  1375. if not toPersonId:
  1376. if debug:
  1377. print('DEBUG: missing personId')
  1378. return 5
  1379. # sharedInbox and capabilities are optional
  1380. # get the senders private key
  1381. privateKeyPem=getPersonKey(nickname,domain,baseDir,'private',debug)
  1382. if len(privateKeyPem)==0:
  1383. if debug:
  1384. print('DEBUG: Private key not found for '+ \
  1385. nickname+'@'+domain+' in '+baseDir+'/keys/private')
  1386. return 6
  1387. if toDomain not in inboxUrl:
  1388. if debug:
  1389. print('DEBUG: '+toDomain+' is not in '+inboxUrl)
  1390. return 7
  1391. postPath=inboxUrl.split(toDomain,1)[1]
  1392. addFollowersToPublicPost(postJsonObject)
  1393. # convert json to string so that there are no
  1394. # subsequent conversions after creating message body digest
  1395. postJsonStr=json.dumps(postJsonObject)
  1396. # construct the http header, including the message body digest
  1397. signatureHeaderJson = \
  1398. createSignedHeader(privateKeyPem,nickname,domain,port, \
  1399. toDomain,toPort, \
  1400. postPath,httpPrefix,withDigest,postJsonStr)
  1401. # Keep the number of threads being used small
  1402. while len(sendThreads)>1000:
  1403. print('WARN: Maximum threads reached - killing send thread')
  1404. sendThreads[0].kill()
  1405. sendThreads.pop(0)
  1406. print('WARN: thread killed')
  1407. if debug:
  1408. print('DEBUG: starting thread to send post')
  1409. pprint(postJsonObject)
  1410. thr = threadWithTrace(target=threadSendPost, \
  1411. args=(session, \
  1412. postJsonStr, \
  1413. federationList, \
  1414. inboxUrl,baseDir, \
  1415. signatureHeaderJson.copy(), \
  1416. postLog,
  1417. debug),daemon=True)
  1418. sendThreads.append(thr)
  1419. #thr.start()
  1420. return 0
  1421. def addToField(activityType: str,postJsonObject: {},debug: bool) -> ({},bool):
  1422. """The Follow activity doesn't have a 'to' field and so one
  1423. needs to be added so that activity distribution happens in a consistent way
  1424. Returns true if a 'to' field exists or was added
  1425. """
  1426. if postJsonObject.get('to'):
  1427. return postJsonObject,True
  1428. if debug:
  1429. pprint(postJsonObject)
  1430. print('DEBUG: no "to" field when sending to named addresses 2')
  1431. isSameType=False
  1432. toFieldAdded=False
  1433. if postJsonObject.get('object'):
  1434. if isinstance(postJsonObject['object'], str):
  1435. if postJsonObject.get('type'):
  1436. if postJsonObject['type']==activityType:
  1437. isSameType=True
  1438. if debug:
  1439. print('DEBUG: "to" field assigned to Follow')
  1440. toAddress=postJsonObject['object']
  1441. if '/statuses/' in toAddress:
  1442. toAddress=toAddress.split('/statuses/')[0]
  1443. postJsonObject['to']=[toAddress]
  1444. toFieldAdded=True
  1445. elif isinstance(postJsonObject['object'], dict):
  1446. if postJsonObject['object'].get('type'):
  1447. if postJsonObject['object']['type']==activityType:
  1448. isSameType=True
  1449. if isinstance(postJsonObject['object']['object'], str):
  1450. if debug:
  1451. print('DEBUG: "to" field assigned to Follow')
  1452. toAddress=postJsonObject['object']['object']
  1453. if '/statuses/' in toAddress:
  1454. toAddress=toAddress.split('/statuses/')[0]
  1455. postJsonObject['object']['to']=[toAddress]
  1456. postJsonObject['to']= \
  1457. [postJsonObject['object']['object']]
  1458. toFieldAdded=True
  1459. if not isSameType:
  1460. return postJsonObject,True
  1461. if toFieldAdded:
  1462. return postJsonObject,True
  1463. return postJsonObject,False
  1464. def sendToNamedAddresses(session,baseDir: str, \
  1465. nickname: str, domain: str, port: int, \
  1466. httpPrefix: str,federationList: [], \
  1467. sendThreads: [],postLog: [], \
  1468. cachedWebfingers: {},personCache: {}, \
  1469. postJsonObject: {},debug: bool, \
  1470. projectVersion: str) -> None:
  1471. """sends a post to the specific named addresses in to/cc
  1472. """
  1473. if not session:
  1474. print('WARN: No session for sendToNamedAddresses')
  1475. return
  1476. if not postJsonObject.get('object'):
  1477. return
  1478. if isinstance(postJsonObject['object'], dict):
  1479. isProfileUpdate=False
  1480. # for actor updates there is no 'to' within the object
  1481. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1482. if postJsonObject['type']=='Update' and \
  1483. (postJsonObject['object']['type']=='Person' or \
  1484. postJsonObject['object']['type']=='Application' or \
  1485. postJsonObject['object']['type']=='Group' or \
  1486. postJsonObject['object']['type']=='Service'):
  1487. # use the original object, which has a 'to'
  1488. recipientsObject=postJsonObject
  1489. isProfileUpdate=True
  1490. if not isProfileUpdate:
  1491. if not postJsonObject['object'].get('to'):
  1492. if debug:
  1493. pprint(postJsonObject)
  1494. print('DEBUG: no "to" field when sending to named addresses')
  1495. if postJsonObject['object'].get('type'):
  1496. if postJsonObject['object']['type']=='Follow':
  1497. if isinstance(postJsonObject['object']['object'], str):
  1498. if debug:
  1499. print('DEBUG: "to" field assigned to Follow')
  1500. postJsonObject['object']['to']= \
  1501. [postJsonObject['object']['object']]
  1502. if not postJsonObject['object'].get('to'):
  1503. return
  1504. recipientsObject=postJsonObject['object']
  1505. else:
  1506. postJsonObject,fieldAdded=addToField('Follow',postJsonObject,debug)
  1507. if not fieldAdded:
  1508. return
  1509. postJsonObject,fieldAdded=addToField('Like',postJsonObject,debug)
  1510. if not fieldAdded:
  1511. return
  1512. recipientsObject=postJsonObject
  1513. recipients=[]
  1514. recipientType=['to','cc']
  1515. for rType in recipientType:
  1516. if not recipientsObject.get(rType):
  1517. continue
  1518. if isinstance(recipientsObject[rType], list):
  1519. if debug:
  1520. pprint(recipientsObject)
  1521. print('recipientsObject: '+str(recipientsObject[rType]))
  1522. for address in recipientsObject[rType]:
  1523. if not address:
  1524. continue
  1525. if '/' not in address:
  1526. continue
  1527. if address.endswith('#Public'):
  1528. continue
  1529. if address.endswith('/followers'):
  1530. continue
  1531. recipients.append(address)
  1532. elif isinstance(recipientsObject[rType], str):
  1533. address=recipientsObject[rType]
  1534. if address:
  1535. if '/' in address:
  1536. if address.endswith('#Public'):
  1537. continue
  1538. if address.endswith('/followers'):
  1539. continue
  1540. recipients.append(address)
  1541. if not recipients:
  1542. if debug:
  1543. print('DEBUG: no individual recipients')
  1544. return
  1545. if debug:
  1546. print('DEBUG: Sending individually addressed posts: '+str(recipients))
  1547. # this is after the message has arrived at the server
  1548. clientToServer=False
  1549. for address in recipients:
  1550. toNickname=getNicknameFromActor(address)
  1551. if not toNickname:
  1552. continue
  1553. toDomain,toPort=getDomainFromActor(address)
  1554. if not toDomain:
  1555. continue
  1556. if debug:
  1557. domainFull=domain
  1558. if port:
  1559. if port!=80 and port!=443:
  1560. if ':' not in domain:
  1561. domainFull=domain+':'+str(port)
  1562. toDomainFull=toDomain
  1563. if toPort:
  1564. if toPort!=80 and toPort!=443:
  1565. if ':' not in toDomain:
  1566. toDomainFull=toDomain+':'+str(toPort)
  1567. print('DEBUG: Post sending s2s: '+nickname+'@'+domainFull+ \
  1568. ' to '+toNickname+'@'+toDomainFull)
  1569. cc=[]
  1570. sendSignedJson(postJsonObject,session,baseDir, \
  1571. nickname,domain,port, \
  1572. toNickname,toDomain,toPort, \
  1573. cc,httpPrefix,True,clientToServer, \
  1574. federationList, \
  1575. sendThreads,postLog,cachedWebfingers, \
  1576. personCache,debug,projectVersion)
  1577. def hasSharedInbox(session,httpPrefix: str,domain: str) -> bool:
  1578. """Returns true if the given domain has a shared inbox
  1579. """
  1580. wfRequest=webfingerHandle(session,domain+'@'+domain,httpPrefix,{}, \
  1581. None,__version__)
  1582. if wfRequest:
  1583. if not wfRequest.get('errors'):
  1584. return True
  1585. return False
  1586. def sendToFollowers(session,baseDir: str, \
  1587. nickname: str, domain: str, port: int, \
  1588. httpPrefix: str,federationList: [], \
  1589. sendThreads: [],postLog: [], \
  1590. cachedWebfingers: {},personCache: {}, \
  1591. postJsonObject: {},debug: bool, \
  1592. projectVersion: str) -> None:
  1593. """sends a post to the followers of the given nickname
  1594. """
  1595. print('sendToFollowers')
  1596. if not session:
  1597. print('WARN: No session for sendToFollowers')
  1598. return
  1599. if not postIsAddressedToFollowers(baseDir,nickname,domain, \
  1600. port,httpPrefix,postJsonObject):
  1601. if debug:
  1602. print('Post is not addressed to followers')
  1603. return
  1604. print('Post is addressed to followers')
  1605. grouped=groupFollowersByDomain(baseDir,nickname,domain)
  1606. if not grouped:
  1607. if debug:
  1608. print('Post to followers did not resolve any domains')
  1609. return
  1610. print('Post to followers resolved domains')
  1611. print(str(grouped))
  1612. # this is after the message has arrived at the server
  1613. clientToServer=False
  1614. # for each instance
  1615. for followerDomain,followerHandles in grouped.items():
  1616. if debug:
  1617. print('DEBUG: follower handles for '+followerDomain)
  1618. pprint(followerHandles)
  1619. withSharedInbox=hasSharedInbox(session,httpPrefix,followerDomain)
  1620. if debug:
  1621. if withSharedInbox:
  1622. print(followerDomain+' has shared inbox')
  1623. else:
  1624. print(followerDomain+' does not have a shared inbox')
  1625. toPort=port
  1626. index=0
  1627. toDomain=followerHandles[index].split('@')[1]
  1628. if ':' in toDomain:
  1629. toPort=toDomain.split(':')[1]
  1630. toDomain=toDomain.split(':')[0]
  1631. cc=''
  1632. if withSharedInbox:
  1633. toNickname=followerHandles[index].split('@')[0]
  1634. # if there are more than one followers on the domain
  1635. # then send the post to the shared inbox
  1636. if len(followerHandles)>1:
  1637. toNickname='inbox'
  1638. if toNickname!='inbox' and postJsonObject.get('type'):
  1639. if postJsonObject['type']=='Update':
  1640. if postJsonObject.get('object'):
  1641. if isinstance(postJsonObject['object'], dict):
  1642. if postJsonObject['object'].get('type'):
  1643. if postJsonObject['object']['type']=='Person' or \
  1644. postJsonObject['object']['type']=='Application' or \
  1645. postJsonObject['object']['type']=='Group' or \
  1646. postJsonObject['object']['type']=='Service':
  1647. print('Sending profile update to shared inbox of '+toDomain)
  1648. toNickname='inbox'
  1649. if debug:
  1650. print('DEBUG: Sending from '+nickname+'@'+domain+ \
  1651. ' to '+toNickname+'@'+toDomain)
  1652. sendSignedJson(postJsonObject,session,baseDir, \
  1653. nickname,domain,port, \
  1654. toNickname,toDomain,toPort, \
  1655. cc,httpPrefix,True,clientToServer, \
  1656. federationList, \
  1657. sendThreads,postLog,cachedWebfingers, \
  1658. personCache,debug,projectVersion)
  1659. else:
  1660. # send to individual followers without using a shared inbox
  1661. for handle in followerHandles:
  1662. if debug:
  1663. print('DEBUG: Sending to '+handle)
  1664. toNickname=handle.split('@')[0]
  1665. if debug:
  1666. if postJsonObject['type']!='Update':
  1667. print('DEBUG: Sending from '+ \
  1668. nickname+'@'+domain+' to '+ \
  1669. toNickname+'@'+toDomain)
  1670. else:
  1671. print('DEBUG: Sending profile update from '+ \
  1672. nickname+'@'+domain+' to '+ \
  1673. toNickname+'@'+toDomain)
  1674. sendSignedJson(postJsonObject,session,baseDir, \
  1675. nickname,domain,port, \
  1676. toNickname,toDomain,toPort, \
  1677. cc,httpPrefix,True,clientToServer, \
  1678. federationList, \
  1679. sendThreads,postLog,cachedWebfingers, \
  1680. personCache,debug,projectVersion)
  1681. time.sleep(4)
  1682. if debug:
  1683. print('DEBUG: End of sendToFollowers')
  1684. def sendToFollowersThread(session,baseDir: str, \
  1685. nickname: str,domain: str,port: int, \
  1686. httpPrefix: str,federationList: [], \
  1687. sendThreads: [],postLog: [], \
  1688. cachedWebfingers: {},personCache: {}, \
  1689. postJsonObject: {},debug: bool, \
  1690. projectVersion: str):
  1691. """Returns a thread used to send a post to followers
  1692. """
  1693. sendThread= \
  1694. threadWithTrace(target=sendToFollowers, \
  1695. args=(session,baseDir, \
  1696. nickname,domain,port, \
  1697. httpPrefix,federationList, \
  1698. sendThreads,postLog, \
  1699. cachedWebfingers,personCache, \
  1700. postJsonObject.copy(),debug, \
  1701. projectVersion),daemon=True)
  1702. sendThread.start()
  1703. return sendThread
  1704. def createInbox(recentPostsCache: {}, \
  1705. session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1706. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1707. return createBoxIndexed(recentPostsCache, \
  1708. session,baseDir,'inbox',nickname,domain,port,httpPrefix, \
  1709. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1710. def createBookmarksTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1711. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1712. return createBoxIndexed({},session,baseDir,'tlbookmarks',nickname,domain,port,httpPrefix, \
  1713. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1714. def createDMTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1715. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1716. return createBoxIndexed({},session,baseDir,'dm',nickname,domain,port,httpPrefix, \
  1717. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1718. def createRepliesTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1719. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1720. return createBoxIndexed({},session,baseDir,'tlreplies',nickname,domain,port,httpPrefix, \
  1721. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1722. def createMediaTimeline(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1723. itemsPerPage: int,headerOnly: bool,ocapAlways: bool,pageNumber=None) -> {}:
  1724. return createBoxIndexed({},session,baseDir,'tlmedia',nickname,domain,port,httpPrefix, \
  1725. itemsPerPage,headerOnly,True,ocapAlways,pageNumber)
  1726. def createOutbox(session,baseDir: str,nickname: str,domain: str,port: int,httpPrefix: str, \
  1727. itemsPerPage: int,headerOnly: bool,authorized: bool,pageNumber=None) -> {}:
  1728. return createBoxIndexed({},session,baseDir,'outbox',nickname,domain,port,httpPrefix, \
  1729. itemsPerPage,headerOnly,authorized,False,pageNumber)
  1730. def createModeration(baseDir: str,nickname: str,domain: str,port: int, \
  1731. httpPrefix: str, \
  1732. itemsPerPage: int,headerOnly: bool, \
  1733. ocapAlways: bool,pageNumber=None) -> {}:
  1734. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1735. boxname='moderation'
  1736. if port:
  1737. if port!=80 and port!=443:
  1738. if ':' not in domain:
  1739. domain=domain+':'+str(port)
  1740. if not pageNumber:
  1741. pageNumber=1
  1742. pageStr='?page='+str(pageNumber)
  1743. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  1744. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1745. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1746. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  1747. 'totalItems': 0,
  1748. 'type': 'OrderedCollection'}
  1749. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  1750. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  1751. 'orderedItems': [
  1752. ],
  1753. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  1754. 'type': 'OrderedCollectionPage'}
  1755. if isModerator(baseDir,nickname):
  1756. moderationIndexFile=baseDir+'/accounts/moderation.txt'
  1757. if os.path.isfile(moderationIndexFile):
  1758. with open(moderationIndexFile, "r") as f:
  1759. lines = f.readlines()
  1760. boxHeader['totalItems']=len(lines)
  1761. if headerOnly:
  1762. return boxHeader
  1763. pageLines=[]
  1764. if len(lines)>0:
  1765. endLineNumber=len(lines)-1-int(itemsPerPage*pageNumber)
  1766. if endLineNumber<0:
  1767. endLineNumber=0
  1768. startLineNumber=len(lines)-1-int(itemsPerPage*(pageNumber-1))
  1769. if startLineNumber<0:
  1770. startLineNumber=0
  1771. lineNumber=startLineNumber
  1772. while lineNumber>=endLineNumber:
  1773. pageLines.append(lines[lineNumber].strip('\n'))
  1774. lineNumber-=1
  1775. for postUrl in pageLines:
  1776. postFilename=boxDir+'/'+postUrl.replace('/','#')+'.json'
  1777. if os.path.isfile(postFilename):
  1778. postJsonObject=loadJson(postFilename)
  1779. if postJsonObject:
  1780. boxItems['orderedItems'].append(postJsonObject)
  1781. if headerOnly:
  1782. return boxHeader
  1783. return boxItems
  1784. def getStatusNumberFromPostFilename(filename) -> int:
  1785. """Gets the status number from a post filename
  1786. eg. https:##testdomain.com:8085#users#testuser567#statuses#1562958506952068.json
  1787. returns 156295850695206
  1788. """
  1789. if '#statuses#' not in filename:
  1790. return None
  1791. return int(filename.split('#')[-1].replace('.json',''))
  1792. def isDM(postJsonObject: {}) -> bool:
  1793. """Returns true if the given post is a DM
  1794. """
  1795. if postJsonObject['type']!='Create':
  1796. return False
  1797. if not postJsonObject.get('object'):
  1798. return False
  1799. if not isinstance(postJsonObject['object'], dict):
  1800. return False
  1801. if postJsonObject['object']['type']!='Note':
  1802. return False
  1803. if postJsonObject['object'].get('moderationStatus'):
  1804. return False
  1805. fields=['to','cc']
  1806. for f in fields:
  1807. if not postJsonObject['object'].get(f):
  1808. continue
  1809. for toAddress in postJsonObject['object'][f]:
  1810. if toAddress.endswith('#Public'):
  1811. return False
  1812. if toAddress.endswith('followers'):
  1813. return False
  1814. return True
  1815. def isImageMedia(session,baseDir: str,httpPrefix: str, \
  1816. nickname: str,domain: str,postJsonObject: {}) -> bool:
  1817. """Returns true if the given post has attached image media
  1818. """
  1819. if postJsonObject['type']=='Announce':
  1820. postJsonAnnounce= \
  1821. downloadAnnounce(session,baseDir,httpPrefix, \
  1822. nickname,domain,postJsonObject,__version__)
  1823. if postJsonAnnounce:
  1824. postJsonObject=postJsonAnnounce
  1825. if postJsonObject['type']!='Create':
  1826. return False
  1827. if not postJsonObject.get('object'):
  1828. return False
  1829. if not isinstance(postJsonObject['object'], dict):
  1830. return False
  1831. if postJsonObject['object'].get('moderationStatus'):
  1832. return False
  1833. if postJsonObject['object']['type']!='Note':
  1834. return False
  1835. if not postJsonObject['object'].get('attachment'):
  1836. return False
  1837. if not isinstance(postJsonObject['object']['attachment'], list):
  1838. return False
  1839. for attach in postJsonObject['object']['attachment']:
  1840. if attach.get('mediaType') and attach.get('url'):
  1841. if attach['mediaType'].startswith('image/') or \
  1842. attach['mediaType'].startswith('audio/') or \
  1843. attach['mediaType'].startswith('video/'):
  1844. return True
  1845. return False
  1846. def isReply(postJsonObject: {},actor: str) -> bool:
  1847. """Returns true if the given post is a reply to the given actor
  1848. """
  1849. if postJsonObject['type']!='Create':
  1850. return False
  1851. if not postJsonObject.get('object'):
  1852. return False
  1853. if not isinstance(postJsonObject['object'], dict):
  1854. return False
  1855. if postJsonObject['object'].get('moderationStatus'):
  1856. return False
  1857. if postJsonObject['object']['type']!='Note':
  1858. return False
  1859. if postJsonObject['object'].get('inReplyTo'):
  1860. if postJsonObject['object']['inReplyTo'].startswith(actor):
  1861. return True
  1862. if not postJsonObject['object'].get('tag'):
  1863. return False
  1864. if not isinstance(postJsonObject['object']['tag'], list):
  1865. return False
  1866. for tag in postJsonObject['object']['tag']:
  1867. if not tag.get('type'):
  1868. continue
  1869. if tag['type']=='Mention':
  1870. if not tag.get('href'):
  1871. continue
  1872. if actor in tag['href']:
  1873. return True
  1874. return False
  1875. def createBoxIndex(boxDir: str,postsInBoxDict: {}) -> int:
  1876. """ Creates an index for the given box
  1877. """
  1878. postsCtr=0
  1879. postsInPersonInbox=os.scandir(boxDir)
  1880. for postFilename in postsInPersonInbox:
  1881. postFilename=postFilename.name
  1882. if not postFilename.endswith('.json'):
  1883. continue
  1884. # extract the status number
  1885. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1886. if statusNumber:
  1887. postsInBoxDict[statusNumber]=os.path.join(boxDir, postFilename)
  1888. postsCtr+=1
  1889. return postsCtr
  1890. def createSharedInboxIndex(baseDir: str,sharedBoxDir: str, \
  1891. postsInBoxDict: {},postsCtr: int, \
  1892. nickname: str,domain: str, \
  1893. ocapAlways: bool) -> int:
  1894. """ Creates an index for the given shared inbox
  1895. """
  1896. handle=nickname+'@'+domain
  1897. followingFilename=baseDir+'/accounts/'+handle+'/following.txt'
  1898. postsInSharedInbox=os.scandir(sharedBoxDir)
  1899. followingHandles=None
  1900. for postFilename in postsInSharedInbox:
  1901. postFilename=postFilename.name
  1902. if not postFilename.endswith('.json'):
  1903. continue
  1904. statusNumber=getStatusNumberFromPostFilename(postFilename)
  1905. if not statusNumber:
  1906. continue
  1907. sharedInboxFilename=os.path.join(sharedBoxDir, postFilename)
  1908. # get the actor from the shared post
  1909. postJsonObject=loadJson(sharedInboxFilename,0)
  1910. if not postJsonObject:
  1911. print('WARN: json load exception createSharedInboxIndex')
  1912. continue
  1913. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  1914. if not actorNickname:
  1915. continue
  1916. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  1917. if not actorDomain:
  1918. continue
  1919. # is the actor followed by this account?
  1920. if not followingHandles:
  1921. with open(followingFilename, 'r') as followingFile:
  1922. followingHandles = followingFile.read()
  1923. if actorNickname+'@'+actorDomain not in followingHandles:
  1924. continue
  1925. if ocapAlways:
  1926. capsList=None
  1927. # Note: should this be in the Create or the object of a post?
  1928. if postJsonObject.get('capability'):
  1929. if isinstance(postJsonObject['capability'], list):
  1930. capsList=postJsonObject['capability']
  1931. # Have capabilities been granted for the sender?
  1932. ocapFilename= \
  1933. baseDir+'/accounts/'+handle+'/ocap/granted/'+ \
  1934. postJsonObject['actor'].replace('/','#')+'.json'
  1935. if not os.path.isfile(ocapFilename):
  1936. continue
  1937. # read the capabilities id
  1938. ocapJson=loadJson(ocapFilename,0)
  1939. if not ocapJson:
  1940. print('WARN: json load exception createSharedInboxIndex')
  1941. else:
  1942. if ocapJson.get('id'):
  1943. if ocapJson['id'] in capsList:
  1944. postsInBoxDict[statusNumber]=sharedInboxFilename
  1945. postsCtr+=1
  1946. else:
  1947. postsInBoxDict[statusNumber]=sharedInboxFilename
  1948. postsCtr+=1
  1949. return postsCtr
  1950. def addPostStringToTimeline(postStr: str,boxname: str, \
  1951. postsInBox: [],boxActor: str) -> bool:
  1952. """ is this a valid timeline post?
  1953. """
  1954. # must be a "Note" or "Announce" type
  1955. if '"Note"' in postStr or '"Announce"' in postStr or \
  1956. ('"Question"' in postStr and ('"Create"' in postStr or '"Update"' in postStr)):
  1957. if boxname=='dm':
  1958. if '#Public' in postStr or '/followers' in postStr:
  1959. return False
  1960. elif boxname=='tlreplies':
  1961. if boxActor not in postStr:
  1962. return False
  1963. elif boxname=='tlmedia':
  1964. if '"Create"' in postStr:
  1965. if 'mediaType' not in postStr or 'image/' not in postStr:
  1966. return False
  1967. # add the post to the dictionary
  1968. postsInBox.append(postStr)
  1969. return True
  1970. return False
  1971. def addPostToTimeline(filePath: str,boxname: str, \
  1972. postsInBox: [],boxActor: str) -> bool:
  1973. """ Reads a post from file and decides whether it is valid
  1974. """
  1975. with open(filePath, 'r') as postFile:
  1976. postStr = postFile.read()
  1977. return addPostStringToTimeline(postStr,boxname,postsInBox,boxActor)
  1978. return False
  1979. def createBoxIndexed(recentPostsCache: {}, \
  1980. session,baseDir: str,boxname: str, \
  1981. nickname: str,domain: str,port: int,httpPrefix: str, \
  1982. itemsPerPage: int,headerOnly: bool,authorized :bool, \
  1983. ocapAlways: bool,pageNumber=None) -> {}:
  1984. """Constructs the box feed for a person with the given nickname
  1985. """
  1986. if not authorized or not pageNumber:
  1987. pageNumber=1
  1988. if boxname!='inbox' and boxname!='dm' and \
  1989. boxname!='tlreplies' and boxname!='tlmedia' and \
  1990. boxname!='outbox' and boxname!='tlbookmarks':
  1991. return None
  1992. if boxname!='dm' and boxname!='tlreplies' and \
  1993. boxname!='tlmedia' and boxname!='tlbookmarks':
  1994. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  1995. else:
  1996. # extract DMs or replies or media from the inbox
  1997. boxDir = createPersonDir(nickname,domain,baseDir,'inbox')
  1998. announceCacheDir=baseDir+'/cache/announce/'+nickname
  1999. sharedBoxDir=None
  2000. if boxname=='inbox' or boxname=='tlreplies' or \
  2001. boxname=='tlmedia':
  2002. sharedBoxDir = createPersonDir('inbox',domain,baseDir,boxname)
  2003. # bookmarks timeline is like the inbox but has its own separate index
  2004. indexBoxName=boxname
  2005. if boxname=='tlbookmarks':
  2006. indexBoxName='bookmarks'
  2007. elif boxname=='dm':
  2008. indexBoxName='dm'
  2009. elif boxname=='tlreplies':
  2010. indexBoxName='tlreplies'
  2011. elif boxname=='tlmedia':
  2012. indexBoxName='tlmedia'
  2013. if port:
  2014. if port!=80 and port!=443:
  2015. if ':' not in domain:
  2016. domain=domain+':'+str(port)
  2017. boxActor=httpPrefix+'://'+domain+'/users/'+nickname
  2018. pageStr='?page=true'
  2019. if pageNumber:
  2020. try:
  2021. pageStr='?page='+str(pageNumber)
  2022. except:
  2023. pass
  2024. boxHeader = {'@context': 'https://www.w3.org/ns/activitystreams',
  2025. 'first': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  2026. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  2027. 'last': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page=true',
  2028. 'totalItems': 0,
  2029. 'type': 'OrderedCollection'}
  2030. boxItems = {'@context': 'https://www.w3.org/ns/activitystreams',
  2031. 'id': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+pageStr,
  2032. 'orderedItems': [
  2033. ],
  2034. 'partOf': httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname,
  2035. 'type': 'OrderedCollectionPage'}
  2036. postsInBox=[]
  2037. indexFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+indexBoxName+'.index'
  2038. postsCtr=0
  2039. if os.path.isfile(indexFilename):
  2040. maxPostCtr=itemsPerPage*pageNumber
  2041. with open(indexFilename, 'r') as indexFile:
  2042. while postsCtr<maxPostCtr:
  2043. postFilename=indexFile.readline()
  2044. if not postFilename:
  2045. postsCtr+=1
  2046. continue
  2047. # Skip through any posts previous to the current page
  2048. if postsCtr<int((pageNumber-1)*itemsPerPage):
  2049. postsCtr+=1
  2050. continue
  2051. # if this is a full path then remove the directories
  2052. if '/' in postFilename:
  2053. postFilename=postFilename.split('/')[-1]
  2054. # filename of the post without any extension or path
  2055. # This should also correspond to any index entry in the posts cache
  2056. postUrl=postFilename.replace('\n','').replace('.json','').strip()
  2057. postAdded=False
  2058. # is the post cached in memory?
  2059. if recentPostsCache.get('index'):
  2060. if postUrl in recentPostsCache['index']:
  2061. if recentPostsCache['json'].get(postUrl):
  2062. addPostStringToTimeline(recentPostsCache['json'][postUrl], \
  2063. boxname,postsInBox,boxActor)
  2064. postAdded=True
  2065. if not postAdded:
  2066. # read the post from file
  2067. fullPostFilename= \
  2068. locatePost(baseDir,nickname,domain,postUrl,False)
  2069. if fullPostFilename:
  2070. addPostToTimeline(fullPostFilename,boxname,postsInBox,boxActor)
  2071. else:
  2072. print('WARN: unable to locate post '+postUrl)
  2073. postsCtr+=1
  2074. # Generate first and last entries within header
  2075. if postsCtr>0:
  2076. lastPage=int(postsCtr/itemsPerPage)
  2077. if lastPage<1:
  2078. lastPage=1
  2079. boxHeader['last']= \
  2080. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+str(lastPage)
  2081. if headerOnly:
  2082. boxHeader['totalItems']=len(postsInBox)
  2083. prevPageStr='true'
  2084. if pageNumber>1:
  2085. prevPageStr=str(pageNumber-1)
  2086. boxHeader['prev']= \
  2087. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+prevPageStr
  2088. nextPageStr=str(pageNumber+1)
  2089. boxHeader['next']= \
  2090. httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname+'?page='+nextPageStr
  2091. return boxHeader
  2092. for postStr in postsInBox:
  2093. p=None
  2094. try:
  2095. p=json.loads(postStr)
  2096. except:
  2097. continue
  2098. # remove any capability so that it's not displayed
  2099. if p.get('capability'):
  2100. del p['capability']
  2101. # Don't show likes, replies or shares (announces) to unauthorized viewers
  2102. if not authorized:
  2103. if p.get('object'):
  2104. if isinstance(p['object'], dict):
  2105. if p['object'].get('likes'):
  2106. p['likes']={'items': []}
  2107. if p['object'].get('replies'):
  2108. p['replies']={}
  2109. if p['object'].get('shares'):
  2110. p['shares']={}
  2111. if p['object'].get('bookmarks'):
  2112. p['bookmarks']={}
  2113. boxItems['orderedItems'].append(p)
  2114. return boxItems
  2115. def expireCache(baseDir: str,personCache: {}, \
  2116. httpPrefix: str,archiveDir: str,maxPostsInBox=32000):
  2117. """Thread used to expire actors from the cache and archive old posts
  2118. """
  2119. while True:
  2120. # once per day
  2121. time.sleep(60*60*24)
  2122. expirePersonCache(basedir,personCache)
  2123. archivePosts(baseDir,httpPrefix,archiveDir,maxPostsInBox)
  2124. def archivePosts(baseDir: str,httpPrefix: str,archiveDir: str, \
  2125. maxPostsInBox=32000) -> None:
  2126. """Archives posts for all accounts
  2127. """
  2128. if archiveDir:
  2129. if not os.path.isdir(archiveDir):
  2130. os.mkdir(archiveDir)
  2131. if archiveDir:
  2132. if not os.path.isdir(archiveDir+'/accounts'):
  2133. os.mkdir(archiveDir+'/accounts')
  2134. for subdir, dirs, files in os.walk(baseDir+'/accounts'):
  2135. for handle in dirs:
  2136. if '@' in handle:
  2137. nickname=handle.split('@')[0]
  2138. domain=handle.split('@')[1]
  2139. archiveSubdir=None
  2140. if archiveDir:
  2141. if not os.path.isdir(archiveDir+'/accounts/'+handle):
  2142. os.mkdir(archiveDir+'/accounts/'+handle)
  2143. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
  2144. os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
  2145. if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
  2146. os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
  2147. archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
  2148. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2149. 'inbox',archiveSubdir, \
  2150. maxPostsInBox)
  2151. if archiveDir:
  2152. archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
  2153. archivePostsForPerson(httpPrefix,nickname,domain,baseDir, \
  2154. 'outbox',archiveSubdir, \
  2155. maxPostsInBox)
  2156. def archivePostsForPerson(httpPrefix: str,nickname: str,domain: str,baseDir: str, \
  2157. boxname: str,archiveDir: str,maxPostsInBox=32000) -> None:
  2158. """Retain a maximum number of posts within the given box
  2159. Move any others to an archive directory
  2160. """
  2161. if boxname!='inbox' and boxname!='outbox':
  2162. return
  2163. if archiveDir:
  2164. if not os.path.isdir(archiveDir):
  2165. os.mkdir(archiveDir)
  2166. boxDir = createPersonDir(nickname,domain,baseDir,boxname)
  2167. postsInBox=os.scandir(boxDir)
  2168. noOfPosts=0
  2169. for f in postsInBox:
  2170. noOfPosts+=1
  2171. if noOfPosts<=maxPostsInBox:
  2172. return
  2173. # remove entries from the index
  2174. handle=nickname+'@'+domain
  2175. indexFilename=baseDir+'/accounts/'+handle+'/'+boxname+'.index'
  2176. if os.path.isfile(indexFilename):
  2177. indexCtr=0
  2178. # get the existing index entries as a string
  2179. newIndex=''
  2180. with open(indexFilename, 'r') as indexFile:
  2181. for postId in indexFile:
  2182. newIndex+=postId
  2183. indexCtr+=1
  2184. if indexCtr>=maxPostsInBox:
  2185. break
  2186. # save the new index file
  2187. if len(newIndex)>0:
  2188. indexFile=open(indexFilename,'w+')
  2189. if indexFile:
  2190. indexFile.write(newIndex)
  2191. indexFile.close()
  2192. postsInBoxDict={}
  2193. postsCtr=0
  2194. postsInBox=os.scandir(boxDir)
  2195. for postFilename in postsInBox:
  2196. postFilename=postFilename.name
  2197. if not postFilename.endswith('.json'):
  2198. continue
  2199. # Time of file creation
  2200. fullFilename=os.path.join(boxDir,postFilename)
  2201. if os.path.isfile(fullFilename):
  2202. content=open(fullFilename).read()
  2203. if '"published":' in content:
  2204. publishedStr=content.split('"published":')[1]
  2205. if '"' in publishedStr:
  2206. publishedStr=publishedStr.split('"')[1]
  2207. if publishedStr.endswith('Z'):
  2208. postsInBoxDict[publishedStr]=postFilename
  2209. postsCtr+=1
  2210. noOfPosts=postsCtr
  2211. if noOfPosts<=maxPostsInBox:
  2212. return
  2213. # sort the list in ascending order of date
  2214. postsInBoxSorted= \
  2215. OrderedDict(sorted(postsInBoxDict.items(),reverse=False))
  2216. # directory containing cached html posts
  2217. postCacheDir=boxDir.replace('/'+boxname,'/postcache')
  2218. for publishedStr,postFilename in postsInBoxSorted.items():
  2219. filePath=os.path.join(boxDir,postFilename)
  2220. if not os.path.isfile(filePath):
  2221. continue
  2222. if archiveDir:
  2223. repliesPath=filePath.replace('.json','.replies')
  2224. archivePath=os.path.join(archiveDir,postFilename)
  2225. os.rename(filePath,archivePath)
  2226. if os.path.isfile(repliesPath):
  2227. os.rename(repliesPath,archivePath)
  2228. else:
  2229. deletePost(baseDir,httpPrefix,nickname,domain,filePath,False)
  2230. # remove cached html posts
  2231. postCacheFilename= \
  2232. os.path.join(postCacheDir,postFilename).replace('.json','.html')
  2233. if os.path.isfile(postCacheFilename):
  2234. os.remove(postCacheFilename)
  2235. noOfPosts-=1
  2236. if noOfPosts<=maxPostsInBox:
  2237. break
  2238. def getPublicPostsOfPerson(baseDir: str,nickname: str,domain: str, \
  2239. raw: bool,simple: bool,useTor: bool, \
  2240. port: int,httpPrefix: str, \
  2241. debug: bool,projectVersion: str) -> None:
  2242. """ This is really just for test purposes
  2243. """
  2244. session = createSession(useTor)
  2245. personCache={}
  2246. cachedWebfingers={}
  2247. federationList=[]
  2248. domainFull=domain
  2249. if port:
  2250. if port!=80 and port!=443:
  2251. if ':' not in domain:
  2252. domainFull=domain+':'+str(port)
  2253. handle=httpPrefix+"://"+domainFull+"/@"+nickname
  2254. wfRequest = \
  2255. webfingerHandle(session,handle,httpPrefix,cachedWebfingers, \
  2256. domain,projectVersion)
  2257. if not wfRequest:
  2258. sys.exit()
  2259. personUrl,pubKeyId,pubKey,personId,shaedInbox,capabilityAcquisition,avatarUrl,displayName= \
  2260. getPersonBox(baseDir,session,wfRequest,personCache, \
  2261. projectVersion,httpPrefix,nickname,domain,'outbox')
  2262. wfResult = json.dumps(wfRequest, indent=2, sort_keys=False)
  2263. maxMentions=10
  2264. maxEmoji=10
  2265. maxAttachments=5
  2266. userPosts = \
  2267. getPosts(session,personUrl,30,maxMentions,maxEmoji, \
  2268. maxAttachments,federationList, \
  2269. personCache,raw,simple,debug, \
  2270. projectVersion,httpPrefix,domain)
  2271. #print(str(userPosts))
  2272. def sendCapabilitiesUpdate(session,baseDir: str,httpPrefix: str, \
  2273. nickname: str,domain: str,port: int, \
  2274. followerUrl,updateCaps: [], \
  2275. sendThreads: [],postLog: [], \
  2276. cachedWebfingers: {},personCache: {}, \
  2277. federationList :[],debug :bool, \
  2278. projectVersion: str) -> int:
  2279. """When the capabilities for a follower are changed this
  2280. sends out an update. followerUrl is the actor of the follower.
  2281. """
  2282. updateJson=capabilitiesUpdate(baseDir,httpPrefix, \
  2283. nickname,domain,port, \
  2284. followerUrl, \
  2285. updateCaps)
  2286. if not updateJson:
  2287. return 1
  2288. if debug:
  2289. pprint(updateJson)
  2290. print('DEBUG: sending capabilities update from '+ \
  2291. nickname+'@'+domain+' port '+ str(port) + \
  2292. ' to '+followerUrl)
  2293. clientToServer=False
  2294. followerNickname=getNicknameFromActor(followerUrl)
  2295. if not followerNickname:
  2296. print('WARN: unable to find nickname in '+followerUrl)
  2297. return 1
  2298. followerDomain,followerPort=getDomainFromActor(followerUrl)
  2299. return sendSignedJson(updateJson,session,baseDir, \
  2300. nickname,domain,port, \
  2301. followerNickname,followerDomain,followerPort, '', \
  2302. httpPrefix,True,clientToServer, \
  2303. federationList, \
  2304. sendThreads,postLog,cachedWebfingers, \
  2305. personCache,debug,projectVersion)
  2306. def populateRepliesJson(baseDir: str,nickname: str,domain: str, \
  2307. postRepliesFilename: str,authorized: bool, \
  2308. repliesJson: {}) -> None:
  2309. # populate the items list with replies
  2310. repliesBoxes=['outbox','inbox']
  2311. with open(postRepliesFilename,'r') as repliesFile:
  2312. for messageId in repliesFile:
  2313. replyFound=False
  2314. # examine inbox and outbox
  2315. for boxname in repliesBoxes:
  2316. searchFilename= \
  2317. baseDir+ \
  2318. '/accounts/'+nickname+'@'+ \
  2319. domain+'/'+ \
  2320. boxname+'/'+ \
  2321. messageId.replace('\n','').replace('/','#')+'.json'
  2322. if os.path.isfile(searchFilename):
  2323. if authorized or \
  2324. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2325. postJsonObject=loadJson(searchFilename)
  2326. if postJsonObject:
  2327. if postJsonObject['object'].get('cc'):
  2328. if authorized or \
  2329. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2330. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2331. repliesJson['orderedItems'].append(postJsonObject)
  2332. replyFound=True
  2333. else:
  2334. if authorized or \
  2335. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2336. repliesJson['orderedItems'].append(postJsonObject)
  2337. replyFound=True
  2338. break
  2339. # if not in either inbox or outbox then examine the shared inbox
  2340. if not replyFound:
  2341. searchFilename= \
  2342. baseDir+ \
  2343. '/accounts/inbox@'+ \
  2344. domain+'/inbox/'+ \
  2345. messageId.replace('\n','').replace('/','#')+'.json'
  2346. if os.path.isfile(searchFilename):
  2347. if authorized or \
  2348. 'https://www.w3.org/ns/activitystreams#Public' in open(searchFilename).read():
  2349. # get the json of the reply and append it to the collection
  2350. postJsonObject=loadJson(searchFilename)
  2351. if postJsonObject:
  2352. if postJsonObject['object'].get('cc'):
  2353. if authorized or \
  2354. ('https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to'] or \
  2355. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['cc']):
  2356. repliesJson['orderedItems'].append(postJsonObject)
  2357. else:
  2358. if authorized or \
  2359. 'https://www.w3.org/ns/activitystreams#Public' in postJsonObject['object']['to']:
  2360. repliesJson['orderedItems'].append(postJsonObject)
  2361. def rejectAnnounce(announceFilename: str):
  2362. """Marks an announce as rejected
  2363. """
  2364. if not os.path.isfile(announceFilename+'.reject'):
  2365. rejectAnnounceFile=open(announceFilename+'.reject', "w+")
  2366. rejectAnnounceFile.write('\n')
  2367. rejectAnnounceFile.close()
  2368. def downloadAnnounce(session,baseDir: str,httpPrefix: str, \
  2369. nickname: str,domain: str, \
  2370. postJsonObject: {},projectVersion: str) -> {}:
  2371. """Download the post referenced by an announce
  2372. """
  2373. if not postJsonObject.get('object'):
  2374. return None
  2375. if not isinstance(postJsonObject['object'], str):
  2376. return None
  2377. # get the announced post
  2378. announceCacheDir=baseDir+'/cache/announce/'+nickname
  2379. if not os.path.isdir(announceCacheDir):
  2380. os.mkdir(announceCacheDir)
  2381. announceFilename= \
  2382. announceCacheDir+'/'+postJsonObject['object'].replace('/','#')+'.json'
  2383. print('announceFilename: '+announceFilename)
  2384. if os.path.isfile(announceFilename+'.reject'):
  2385. return None
  2386. if os.path.isfile(announceFilename):
  2387. print('Reading cached Announce content for '+postJsonObject['object'])
  2388. postJsonObject=loadJson(announceFilename)
  2389. if postJsonObject:
  2390. return postJsonObject
  2391. else:
  2392. print('Downloading Announce content for '+postJsonObject['object'])
  2393. asHeader={'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  2394. if '/channel/' in postJsonObject['actor']:
  2395. asHeader={'Accept': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'}
  2396. actorNickname=getNicknameFromActor(postJsonObject['actor'])
  2397. actorDomain,actorPort=getDomainFromActor(postJsonObject['actor'])
  2398. announcedJson= \
  2399. getJson(session,postJsonObject['object'],asHeader, \
  2400. None,projectVersion,httpPrefix,domain)
  2401. if not announcedJson:
  2402. return None
  2403. if not isinstance(announcedJson, dict):
  2404. print('WARN: announce json is not a dict - '+postJsonObject['object'])
  2405. rejectAnnounce(announceFilename)
  2406. return None
  2407. if not announcedJson.get('id'):
  2408. rejectAnnounce(announceFilename)
  2409. return None
  2410. if '/statuses/' not in announcedJson['id']:
  2411. rejectAnnounce(announceFilename)
  2412. return None
  2413. if '/users/' not in announcedJson['id'] and \
  2414. '/channel/' not in announcedJson['id'] and \
  2415. '/profile/' not in announcedJson['id']:
  2416. rejectAnnounce(announceFilename)
  2417. return None
  2418. if not announcedJson.get('type'):
  2419. rejectAnnounce(announceFilename)
  2420. #pprint(announcedJson)
  2421. return None
  2422. if announcedJson['type']!='Note':
  2423. rejectAnnounce(announceFilename)
  2424. #pprint(announcedJson)
  2425. return None
  2426. # wrap in create to be consistent with other posts
  2427. announcedJson= \
  2428. outboxMessageCreateWrap(httpPrefix, \
  2429. actorNickname,actorDomain,actorPort, \
  2430. announcedJson)
  2431. if announcedJson['type']!='Create':
  2432. rejectAnnounce(announceFilename)
  2433. #pprint(announcedJson)
  2434. return None
  2435. # set the id to the original status
  2436. announcedJson['id']=postJsonObject['object']
  2437. announcedJson['object']['id']=postJsonObject['object']
  2438. # check that the repeat isn't for a blocked account
  2439. attributedNickname= \
  2440. getNicknameFromActor(announcedJson['object']['id'])
  2441. attributedDomain,attributedPort= \
  2442. getDomainFromActor(announcedJson['object']['id'])
  2443. if attributedNickname and attributedDomain:
  2444. if attributedPort:
  2445. if attributedPort!=80 and attributedPort!=443:
  2446. attributedDomain=attributedDomain+':'+str(attributedPort)
  2447. if isBlocked(baseDir,nickname,domain, \
  2448. attributedNickname,attributedDomain):
  2449. rejectAnnounce(announceFilename)
  2450. return None
  2451. postJsonObject=announcedJson
  2452. if saveJson(postJsonObject,announceFilename):
  2453. return postJsonObject
  2454. return None
  2455. def mutePost(baseDir: str,nickname: str,domain: str,postId: str, \
  2456. recentPostsCache: {}) -> None:
  2457. """ Mutes the given post
  2458. """
  2459. postFilename=locatePost(baseDir,nickname,domain,postId)
  2460. if not postFilename:
  2461. return
  2462. postJsonObject=loadJson(postFilename)
  2463. if not postJsonObject:
  2464. return
  2465. print('MUTE: '+postFilename)
  2466. muteFile=open(postFilename+'.muted', "w")
  2467. if muteFile:
  2468. muteFile.write('\n')
  2469. muteFile.close()
  2470. # remove cached posts so that the muted version gets created
  2471. cachedPostFilename= \
  2472. getCachedPostFilename(baseDir,nickname,domain,postJsonObject)
  2473. if cachedPostFilename:
  2474. if os.path.isfile(cachedPostFilename):
  2475. os.remove(cachedPostFilename)
  2476. # if the post is in the recent posts cache then mark it as muted
  2477. if recentPostsCache.get('index'):
  2478. postId=postJsonObject['id'].replace('/activity','').replace('/','#')
  2479. if postId in recentPostsCache['index']:
  2480. print('MUTE: '+postId+' is in recent posts cache')
  2481. if recentPostsCache['json'].get(postId):
  2482. postJsonObject['muted']=True
  2483. recentPostsCache['json'][postId]=json.dumps(postJsonObject)
  2484. print('MUTE: '+postId+' marked as muted in recent posts cache')
  2485. def unmutePost(baseDir: str,nickname: str,domain: str,postId: str, \
  2486. recentPostsCache: {}) -> None:
  2487. """ Unmutes the given post
  2488. """
  2489. postFilename=locatePost(baseDir,nickname,domain,postId)
  2490. if not postFilename:
  2491. return
  2492. postJsonObject=loadJson(postFilename)
  2493. if not postJsonObject:
  2494. return
  2495. print('UNMUTE: '+postFilename)
  2496. muteFilename=postFilename+'.muted'
  2497. if os.path.isfile(muteFilename):
  2498. os.remove(muteFilename)
  2499. # remove cached posts so that it gets recreated
  2500. cachedPostFilename= \
  2501. getCachedPostFilename(baseDir,nickname,domain,postJsonObject)
  2502. if cachedPostFilename:
  2503. if os.path.isfile(cachedPostFilename):
  2504. os.remove(cachedPostFilename)
  2505. removePostFromCache(postJsonObject,recentPostsCache)