2024-07-07 16:59:12 +00:00
|
|
|
{- P2P protocol over HTTP
|
|
|
|
-
|
|
|
|
- https://git-annex.branchable.com/design/p2p_protocol_over_http/
|
|
|
|
-
|
|
|
|
- Copyright 2024 Joey Hess <id@joeyh.name>
|
|
|
|
-
|
|
|
|
- Licensed under the GNU AGPL version 3 or higher.
|
|
|
|
-}
|
|
|
|
|
|
|
|
{-# LANGUAGE DataKinds #-}
|
|
|
|
{-# LANGUAGE TypeOperators #-}
|
2024-07-08 01:20:50 +00:00
|
|
|
{-# LANGUAGE TypeFamilies #-}
|
|
|
|
{-# LANGUAGE FlexibleInstances #-}
|
|
|
|
{-# LANGUAGE DeriveAnyClass #-}
|
|
|
|
{-# LANGUAGE DeriveGeneric #-}
|
2024-07-07 18:48:20 +00:00
|
|
|
{-# LANGUAGE MultiParamTypeClasses #-}
|
2024-07-07 16:59:12 +00:00
|
|
|
{-# LANGUAGE OverloadedStrings #-}
|
|
|
|
|
|
|
|
module P2P.Http where
|
|
|
|
|
|
|
|
import Annex.Common
|
|
|
|
import qualified P2P.Protocol as P2P
|
|
|
|
import Utility.Base64
|
|
|
|
import Utility.MonotonicClock
|
|
|
|
|
|
|
|
import Servant
|
2024-07-07 20:08:05 +00:00
|
|
|
import Servant.Client.Streaming
|
2024-07-07 18:48:20 +00:00
|
|
|
import qualified Servant.Types.SourceT as S
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
import Network.HTTP.Client (defaultManagerSettings, newManager)
|
2024-07-07 18:48:20 +00:00
|
|
|
import qualified Data.Text as T
|
2024-07-07 16:59:12 +00:00
|
|
|
import qualified Data.Text.Encoding as TE
|
|
|
|
import qualified Data.ByteString as B
|
2024-07-08 01:20:50 +00:00
|
|
|
import Text.Read (readMaybe)
|
2024-07-07 18:48:20 +00:00
|
|
|
import Data.Aeson hiding (Key)
|
2024-07-08 01:20:50 +00:00
|
|
|
import Control.DeepSeq
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
import Control.Concurrent
|
|
|
|
import Control.Concurrent.STM
|
2024-07-08 01:20:50 +00:00
|
|
|
import GHC.Generics
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
type P2PHttpAPI
|
2024-07-07 20:08:05 +00:00
|
|
|
= "git-annex" :> "v3" :> "key" :> CaptureKey
|
2024-07-07 16:59:12 +00:00
|
|
|
:> GetAPI '[DataLengthHeader]
|
|
|
|
:<|> "git-annex" :> "v2" :> "key" :> CaptureKey
|
|
|
|
:> GetAPI '[DataLengthHeader]
|
|
|
|
:<|> "git-annex" :> "v1" :> "key" :> CaptureKey
|
|
|
|
:> GetAPI '[DataLengthHeader]
|
|
|
|
:<|> "git-annex" :> "v0" :> "key" :> CaptureKey
|
|
|
|
:> GetAPI '[]
|
|
|
|
:<|> "git-annex" :> "v3" :> "checkpresent" :> CheckPresentAPI
|
|
|
|
:<|> "git-annex" :> "v2" :> "checkpresent" :> CheckPresentAPI
|
|
|
|
:<|> "git-annex" :> "v1" :> "checkpresent" :> CheckPresentAPI
|
|
|
|
:<|> "git-annex" :> "v0" :> "checkpresent" :> CheckPresentAPI
|
|
|
|
:<|> "git-annex" :> "v3" :> "remove" :> RemoveAPI RemoveResultPlus
|
|
|
|
:<|> "git-annex" :> "v2" :> "remove" :> RemoveAPI RemoveResultPlus
|
|
|
|
:<|> "git-annex" :> "v1" :> "remove" :> RemoveAPI RemoveResult
|
|
|
|
:<|> "git-annex" :> "v0" :> "remove" :> RemoveAPI RemoveResult
|
|
|
|
:<|> "git-annex" :> "v3" :> "remove-before" :> RemoveBeforeAPI
|
|
|
|
:<|> "git-annex" :> "v3" :> "gettimestamp" :> GetTimestampAPI
|
|
|
|
:<|> "git-annex" :> "v3" :> "put" :> DataLengthHeader
|
|
|
|
:> PutAPI PutResultPlus
|
|
|
|
:<|> "git-annex" :> "v2" :> "put" :> DataLengthHeader
|
|
|
|
:> PutAPI PutResultPlus
|
|
|
|
:<|> "git-annex" :> "v1" :> "put" :> DataLengthHeader
|
|
|
|
:> PutAPI PutResult
|
|
|
|
:<|> "git-annex" :> "v0" :> "put"
|
|
|
|
:> PutAPI PutResult
|
|
|
|
:<|> "git-annex" :> "v3" :> "putoffset"
|
|
|
|
:> PutOffsetAPI PutOffsetResultPlus
|
|
|
|
:<|> "git-annex" :> "v2" :> "putoffset"
|
|
|
|
:> PutOffsetAPI PutOffsetResultPlus
|
|
|
|
:<|> "git-annex" :> "v1" :> "putoffset"
|
|
|
|
:> PutOffsetAPI PutOffsetResult
|
2024-07-08 01:20:50 +00:00
|
|
|
:<|> "git-annex" :> "v3" :> "lockcontent" :> LockContentAPI
|
|
|
|
:<|> "git-annex" :> "v2" :> "lockcontent" :> LockContentAPI
|
|
|
|
:<|> "git-annex" :> "v1" :> "lockcontent" :> LockContentAPI
|
|
|
|
:<|> "git-annex" :> "v0" :> "lockcontent" :> LockContentAPI
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
:<|> "git-annex" :> "v3" :> "keeplocked" :> KeepLockedAPI
|
|
|
|
:<|> "git-annex" :> "v2" :> "keeplocked" :> KeepLockedAPI
|
|
|
|
:<|> "git-annex" :> "v1" :> "keeplocked" :> KeepLockedAPI
|
|
|
|
:<|> "git-annex" :> "v0" :> "keeplocked" :> KeepLockedAPI
|
2024-07-07 20:08:05 +00:00
|
|
|
:<|> "git-annex" :> "key" :> CaptureKey :> GetAPI '[]
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
p2pHttpAPI :: Proxy P2PHttpAPI
|
|
|
|
p2pHttpAPI = Proxy
|
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
p2pHttpApp :: Application
|
|
|
|
p2pHttpApp = serve p2pHttpAPI serveP2pHttp
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
serveP2pHttp :: Server P2PHttpAPI
|
|
|
|
serveP2pHttp
|
2024-07-07 20:08:05 +00:00
|
|
|
= serveGet
|
2024-07-07 18:48:20 +00:00
|
|
|
:<|> serveGet
|
|
|
|
:<|> serveGet
|
|
|
|
:<|> serveGet0
|
|
|
|
:<|> serveCheckPresent
|
|
|
|
:<|> serveCheckPresent
|
|
|
|
:<|> serveCheckPresent
|
|
|
|
:<|> serveCheckPresent
|
|
|
|
:<|> serveRemove id
|
|
|
|
:<|> serveRemove id
|
|
|
|
:<|> serveRemove dePlus
|
|
|
|
:<|> serveRemove dePlus
|
|
|
|
:<|> serveRemoveBefore
|
|
|
|
:<|> serveGetTimestamp
|
|
|
|
:<|> servePut id
|
|
|
|
:<|> servePut id
|
|
|
|
:<|> servePut dePlus
|
2024-07-07 20:08:05 +00:00
|
|
|
:<|> servePut dePlus Nothing
|
2024-07-07 18:48:20 +00:00
|
|
|
:<|> servePutOffset id
|
|
|
|
:<|> servePutOffset id
|
|
|
|
:<|> servePutOffset dePlus
|
2024-07-08 01:20:50 +00:00
|
|
|
:<|> serveLockContent
|
|
|
|
:<|> serveLockContent
|
|
|
|
:<|> serveLockContent
|
|
|
|
:<|> serveLockContent
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
:<|> serveKeepLocked
|
|
|
|
:<|> serveKeepLocked
|
|
|
|
:<|> serveKeepLocked
|
|
|
|
:<|> serveKeepLocked
|
2024-07-07 20:08:05 +00:00
|
|
|
:<|> serveGet0
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type GetAPI headers
|
2024-07-07 18:48:20 +00:00
|
|
|
= ClientUUID Optional
|
|
|
|
:> ServerUUID Optional
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> AssociatedFileParam
|
|
|
|
:> OffsetParam
|
|
|
|
:> StreamGet NoFraming OctetStream
|
|
|
|
(Headers headers (SourceIO B.ByteString))
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
serveGet
|
|
|
|
:: B64Key
|
|
|
|
-> Maybe (B64UUID ClientSide)
|
|
|
|
-> Maybe (B64UUID ServerSide)
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe B64FilePath
|
|
|
|
-> Maybe Offset
|
|
|
|
-> Handler (Headers '[DataLengthHeader] (S.SourceT IO B.ByteString))
|
|
|
|
serveGet = undefined
|
|
|
|
|
|
|
|
serveGet0
|
|
|
|
:: B64Key
|
|
|
|
-> Maybe (B64UUID ClientSide)
|
|
|
|
-> Maybe (B64UUID ServerSide)
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe B64FilePath
|
|
|
|
-> Maybe Offset
|
|
|
|
-> Handler (Headers '[] (S.SourceT IO B.ByteString))
|
|
|
|
serveGet0 = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientGet
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> Maybe (B64UUID ClientSide)
|
|
|
|
-> Maybe (B64UUID ServerSide)
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe B64FilePath
|
|
|
|
-> Maybe Offset
|
|
|
|
-> ClientM (Headers '[DataLengthHeader] (S.SourceT IO B.ByteString))
|
|
|
|
clientGet (P2P.ProtocolVersion ver) k cu su bypass af o = case ver of
|
|
|
|
3 -> v3 k cu su bypass af o
|
|
|
|
2 -> v2 k cu su bypass af o
|
|
|
|
1 -> error "XXX" -- TODO v1
|
|
|
|
0 -> error "XXX" -- TODO v0
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type CheckPresentAPI
|
|
|
|
= KeyParam
|
2024-07-07 18:48:20 +00:00
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> Post '[JSON] CheckPresentResult
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
serveCheckPresent
|
|
|
|
:: B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Handler CheckPresentResult
|
|
|
|
serveCheckPresent = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientCheckPresent
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> ClientM CheckPresentResult
|
|
|
|
clientCheckPresent (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
2 -> v2
|
|
|
|
1 -> v1
|
|
|
|
0 -> v0
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type RemoveAPI result
|
|
|
|
= KeyParam
|
2024-07-07 18:48:20 +00:00
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> Post '[JSON] result
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
serveRemove
|
|
|
|
:: (RemoveResultPlus -> t)
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Handler t
|
|
|
|
serveRemove = undefined
|
2024-07-07 20:08:05 +00:00
|
|
|
|
|
|
|
clientRemove
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> ClientM RemoveResultPlus
|
|
|
|
clientRemove (P2P.ProtocolVersion ver) k cu su bypass = case ver of
|
|
|
|
3 -> v3 k cu su bypass
|
|
|
|
2 -> v2 k cu su bypass
|
|
|
|
1 -> plus <$> v1 k cu su bypass
|
|
|
|
0 -> plus <$> v0 k cu su bypass
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type RemoveBeforeAPI
|
|
|
|
= KeyParam
|
2024-07-07 18:48:20 +00:00
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
|
|
|
:> QueryParam' '[Required] "timestamp" Timestamp
|
2024-07-07 16:59:12 +00:00
|
|
|
:> Post '[JSON] RemoveResult
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
serveRemoveBefore
|
|
|
|
:: B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Timestamp
|
|
|
|
-> Handler RemoveResult
|
|
|
|
serveRemoveBefore = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientRemoveBefore
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Timestamp
|
|
|
|
-> ClientM RemoveResult
|
|
|
|
clientRemoveBefore (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> _ = client p2pHttpAPI
|
|
|
|
|
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type GetTimestampAPI
|
2024-07-07 18:48:20 +00:00
|
|
|
= ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> Post '[JSON] GetTimestampResult
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
serveGetTimestamp
|
|
|
|
:: B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Handler GetTimestampResult
|
|
|
|
serveGetTimestamp = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientGetTimestamp
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> ClientM GetTimestampResult
|
|
|
|
clientGetTimestamp (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|>
|
|
|
|
v3 :<|> _ = client p2pHttpAPI
|
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type PutAPI result
|
|
|
|
= KeyParam
|
2024-07-07 18:48:20 +00:00
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> AssociatedFileParam
|
|
|
|
:> OffsetParam
|
|
|
|
:> Header' '[Required] "X-git-annex-data-length" DataLength
|
|
|
|
:> StreamBody NoFraming OctetStream (SourceIO B.ByteString)
|
|
|
|
:> Post '[JSON] result
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
servePut
|
|
|
|
:: (PutResultPlus -> t)
|
|
|
|
-> Maybe Integer
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe B64FilePath
|
|
|
|
-> Maybe Offset
|
|
|
|
-> DataLength
|
|
|
|
-> S.SourceT IO B.ByteString
|
|
|
|
-> Handler t
|
|
|
|
servePut = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientPut
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> Maybe Integer
|
|
|
|
-> B64Key
|
2024-07-07 18:48:20 +00:00
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe B64FilePath
|
|
|
|
-> Maybe Offset
|
|
|
|
-> DataLength
|
|
|
|
-> S.SourceT IO B.ByteString
|
2024-07-07 20:08:05 +00:00
|
|
|
-> ClientM PutResultPlus
|
|
|
|
clientPut (P2P.ProtocolVersion ver) sz k cu su bypass af o l src = case ver of
|
|
|
|
3 -> v3 sz k cu su bypass af o l src
|
|
|
|
2 -> v2 sz k cu su bypass af o l src
|
|
|
|
1 -> plus <$> v1 sz k cu su bypass af o l src
|
|
|
|
0 -> plus <$> v0 k cu su bypass af o l src
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|>
|
|
|
|
v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
type PutOffsetAPI result
|
|
|
|
= KeyParam
|
2024-07-07 18:48:20 +00:00
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
2024-07-07 16:59:12 +00:00
|
|
|
:> Post '[JSON] result
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
servePutOffset
|
|
|
|
:: (PutOffsetResultPlus -> t)
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Handler t
|
|
|
|
servePutOffset = undefined
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
clientPutOffset
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> ClientM PutOffsetResultPlus
|
|
|
|
clientPutOffset (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
2 -> v2
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> v2 :<|> _ = client p2pHttpAPI
|
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
type LockContentAPI
|
|
|
|
= KeyParam
|
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
:> Post '[JSON] LockResult
|
2024-07-07 20:08:05 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
serveLockContent
|
|
|
|
:: B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
-> Handler LockResult
|
2024-07-08 01:20:50 +00:00
|
|
|
serveLockContent = undefined
|
2024-07-07 18:48:20 +00:00
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
clientLockContent
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> ClientM LockResult
|
|
|
|
clientLockContent (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
2 -> v2
|
|
|
|
1 -> v1
|
|
|
|
0 -> v0
|
|
|
|
_ -> error "unsupported protocol version"
|
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
2024-07-07 18:48:20 +00:00
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
type KeepLockedAPI
|
|
|
|
= KeyParam
|
|
|
|
:> ClientUUID Required
|
|
|
|
:> ServerUUID Required
|
|
|
|
:> BypassUUIDs
|
|
|
|
:> Header "Connection" ConnectionKeepAlive
|
|
|
|
:> Header "Keep-Alive" KeepAlive
|
|
|
|
:> StreamBody NewlineFraming JSON (SourceIO UnlockRequest)
|
|
|
|
:> Post '[JSON] LockResult
|
2024-07-07 18:48:20 +00:00
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
serveKeepLocked
|
2024-07-08 01:20:50 +00:00
|
|
|
:: B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
-> Maybe ConnectionKeepAlive
|
|
|
|
-> Maybe KeepAlive
|
|
|
|
-> S.SourceT IO UnlockRequest
|
|
|
|
-> Handler LockResult
|
|
|
|
serveKeepLocked k cu su _ _ _ unlockrequeststream = do
|
|
|
|
_ <- liftIO $ S.unSourceT unlockrequeststream go
|
|
|
|
return (LockResult False)
|
|
|
|
where
|
|
|
|
go S.Stop = do
|
|
|
|
print "lost connection to client, drop lock here" -- XXX TODO
|
|
|
|
go (S.Error err) = do
|
|
|
|
print ("Error", err)
|
|
|
|
print "error, drop lock here" -- XXX TODO
|
|
|
|
go (S.Skip s) = go s
|
|
|
|
go (S.Effect ms) = ms >>= go
|
|
|
|
go (S.Yield (UnlockRequest False) s) = go s
|
|
|
|
go (S.Yield (UnlockRequest True) _) = do
|
|
|
|
print ("got unlock request, drop lock here") -- XXX TODO
|
|
|
|
|
|
|
|
clientKeepLocked
|
|
|
|
:: P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> Maybe ConnectionKeepAlive
|
|
|
|
-> Maybe KeepAlive
|
|
|
|
-> S.SourceT IO UnlockRequest
|
|
|
|
-> ClientM LockResult
|
|
|
|
clientKeepLocked (P2P.ProtocolVersion ver) = case ver of
|
|
|
|
3 -> v3
|
|
|
|
2 -> v2
|
|
|
|
1 -> v1
|
|
|
|
0 -> v0
|
|
|
|
_ -> error "unsupported protocol version"
|
2024-07-08 01:20:50 +00:00
|
|
|
where
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
_ :<|> _ :<|> _ :<|>
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
_ :<|> _ :<|> _ :<|> _ :<|>
|
|
|
|
v3 :<|> v2 :<|> v1 :<|> v0 :<|> _ = client p2pHttpAPI
|
2024-07-08 01:20:50 +00:00
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
clientKeepLocked'
|
|
|
|
:: ClientEnv
|
|
|
|
-> P2P.ProtocolVersion
|
|
|
|
-> B64Key
|
|
|
|
-> B64UUID ClientSide
|
|
|
|
-> B64UUID ServerSide
|
|
|
|
-> [B64UUID Bypass]
|
|
|
|
-> TMVar Bool
|
|
|
|
-> IO ()
|
|
|
|
clientKeepLocked' clientenv protover key cu su bypass keeplocked = do
|
|
|
|
let cli = clientKeepLocked protover key cu su bypass
|
|
|
|
(Just connectionKeepAlive) (Just keepAlive)
|
|
|
|
(S.fromStepT unlocksender)
|
|
|
|
withClientM cli clientenv $ \case
|
|
|
|
Left err -> throwM err
|
|
|
|
Right (LockResult _) ->
|
|
|
|
liftIO $ print "end of lock connection to server"
|
|
|
|
where
|
|
|
|
unlocksender =
|
|
|
|
S.Yield (UnlockRequest False) $ S.Effect $ do
|
|
|
|
liftIO $ print "sent keep locked request"
|
|
|
|
return $ S.Effect $ do
|
|
|
|
stilllock <- liftIO $ atomically $ takeTMVar keeplocked
|
|
|
|
if stilllock
|
|
|
|
then return unlocksender
|
|
|
|
else do
|
|
|
|
liftIO $ print "sending unlock request"
|
|
|
|
return $ S.Yield (UnlockRequest True) S.Stop
|
|
|
|
|
|
|
|
testClientLock = do
|
|
|
|
mgr <- newManager defaultManagerSettings
|
|
|
|
burl <- parseBaseUrl "http://localhost:8080/"
|
|
|
|
keeplocked <- newEmptyTMVarIO
|
|
|
|
_ <- forkIO $ do
|
|
|
|
print "running, press enter to drop lock"
|
|
|
|
_ <- getLine
|
|
|
|
atomically $ writeTMVar keeplocked False
|
|
|
|
clientKeepLocked' (mkClientEnv mgr burl)
|
|
|
|
(P2P.ProtocolVersion 3)
|
2024-07-08 01:20:50 +00:00
|
|
|
(B64Key (fromJust $ deserializeKey "WORM--foo"))
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
(B64UUID (toUUID ("cu" :: String)))
|
|
|
|
(B64UUID (toUUID ("su" :: String)))
|
2024-07-08 01:20:50 +00:00
|
|
|
[]
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
keeplocked
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
type ClientUUID req = QueryParam' '[req] "clientuuid" (B64UUID ClientSide)
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
type ServerUUID req = QueryParam' '[req] "serveruuid" (B64UUID ServerSide)
|
|
|
|
|
|
|
|
type BypassUUIDs = QueryParams "bypass" (B64UUID Bypass)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
type CaptureKey = Capture "key" B64Key
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
type KeyParam = QueryParam' '[Required] "key" B64Key
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
type AssociatedFileParam = QueryParam "associatedfile" B64FilePath
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
type OffsetParam = QueryParam "offset" Offset
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
type DataLengthHeader = Header "X-git-annex-data-length" Integer
|
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
-- Phantom types for B64UIID
|
|
|
|
data ClientSide
|
|
|
|
data ServerSide
|
|
|
|
data Bypass
|
|
|
|
data Plus
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
-- Keys, UUIDs, and filenames are base64 encoded since Servant uses
|
|
|
|
-- Text and so needs UTF-8.
|
|
|
|
newtype B64Key = B64Key Key
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
newtype B64UUID t = B64UUID UUID
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show, Generic, NFData)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
newtype B64FilePath = B64FilePath RawFilePath
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
newtype DataLength = DataLength Integer
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
newtype CheckPresentResult = CheckPresentResult Bool
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
newtype RemoveResult = RemoveResult Bool
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
data RemoveResultPlus = RemoveResultPlus Bool [B64UUID Plus]
|
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
newtype GetTimestampResult = GetTimestampResult Timestamp
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
newtype PutResult = PutResult Bool
|
2024-07-07 18:48:20 +00:00
|
|
|
deriving (Eq, Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
data PutResultPlus = PutResultPlus Bool [B64UUID Plus]
|
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
newtype PutOffsetResult = PutOffsetResult Offset
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
data PutOffsetResultPlus = PutOffsetResultPlus Offset [B64UUID Plus]
|
|
|
|
deriving (Show, Generic, NFData)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
newtype Offset = Offset P2P.Offset
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show, Generic, NFData)
|
2024-07-07 18:48:20 +00:00
|
|
|
|
|
|
|
newtype Timestamp = Timestamp MonotonicTimestamp
|
2024-07-08 01:20:50 +00:00
|
|
|
deriving (Show)
|
2024-07-07 20:08:05 +00:00
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
newtype LockResult = LockResult Bool
|
|
|
|
deriving (Show, Generic, NFData)
|
|
|
|
|
|
|
|
newtype UnlockRequest = UnlockRequest Bool
|
|
|
|
deriving (Show, Generic, NFData)
|
|
|
|
|
|
|
|
newtype ConnectionKeepAlive = ConnectionKeepAlive T.Text
|
|
|
|
|
|
|
|
connectionKeepAlive :: ConnectionKeepAlive
|
|
|
|
connectionKeepAlive = ConnectionKeepAlive "Keep-Alive"
|
|
|
|
|
|
|
|
newtype KeepAlive = KeepAlive T.Text
|
|
|
|
|
|
|
|
keepAlive :: KeepAlive
|
|
|
|
keepAlive = KeepAlive "timeout=1200"
|
|
|
|
|
|
|
|
instance ToHttpApiData ConnectionKeepAlive where
|
|
|
|
toUrlPiece (ConnectionKeepAlive t) = t
|
|
|
|
|
|
|
|
instance FromHttpApiData ConnectionKeepAlive where
|
|
|
|
parseUrlPiece = Right . ConnectionKeepAlive
|
|
|
|
|
|
|
|
instance ToHttpApiData KeepAlive where
|
|
|
|
toUrlPiece (KeepAlive t) = t
|
|
|
|
|
|
|
|
instance FromHttpApiData KeepAlive where
|
|
|
|
parseUrlPiece = Right . KeepAlive
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData B64Key where
|
|
|
|
toUrlPiece (B64Key k) = TE.decodeUtf8Lenient $
|
|
|
|
toB64 (serializeKey' k)
|
2024-07-07 16:59:12 +00:00
|
|
|
|
|
|
|
instance FromHttpApiData B64Key where
|
|
|
|
parseUrlPiece t = case fromB64Maybe (TE.encodeUtf8 t) of
|
|
|
|
Nothing -> Left "unable to base64 decode key"
|
|
|
|
Just b -> maybe (Left "key parse error") (Right . B64Key)
|
|
|
|
(deserializeKey' b)
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData (B64UUID t) where
|
|
|
|
toUrlPiece (B64UUID u) = TE.decodeUtf8Lenient $
|
|
|
|
toB64 (fromUUID u)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
instance FromHttpApiData (B64UUID t) where
|
2024-07-07 16:59:12 +00:00
|
|
|
parseUrlPiece t = case fromB64Maybe (TE.encodeUtf8 t) of
|
|
|
|
Nothing -> Left "unable to base64 decode UUID"
|
|
|
|
Just b -> case toUUID b of
|
|
|
|
u@(UUID _) -> Right (B64UUID u)
|
|
|
|
NoUUID -> Left "empty UUID"
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData B64FilePath where
|
|
|
|
toUrlPiece (B64FilePath f) = TE.decodeUtf8Lenient $ toB64 f
|
|
|
|
|
2024-07-07 16:59:12 +00:00
|
|
|
instance FromHttpApiData B64FilePath where
|
|
|
|
parseUrlPiece t = case fromB64Maybe (TE.encodeUtf8 t) of
|
|
|
|
Nothing -> Left "unable to base64 decode filename"
|
|
|
|
Just b -> Right (B64FilePath b)
|
2024-07-07 18:48:20 +00:00
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData Offset where
|
|
|
|
toUrlPiece (Offset (P2P.Offset n)) = T.pack (show n)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
instance FromHttpApiData Offset where
|
|
|
|
parseUrlPiece t = case readMaybe (T.unpack t) of
|
|
|
|
Nothing -> Left "offset parse error"
|
|
|
|
Just n -> Right (Offset (P2P.Offset n))
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData Timestamp where
|
|
|
|
toUrlPiece (Timestamp (MonotonicTimestamp n)) = T.pack (show n)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
instance FromHttpApiData Timestamp where
|
|
|
|
parseUrlPiece t = case readMaybe (T.unpack t) of
|
|
|
|
Nothing -> Left "timestamp parse error"
|
|
|
|
Just n -> Right (Timestamp (MonotonicTimestamp n))
|
|
|
|
|
2024-07-07 20:08:05 +00:00
|
|
|
instance ToHttpApiData DataLength where
|
|
|
|
toUrlPiece (DataLength n) = T.pack (show n)
|
|
|
|
|
2024-07-07 18:48:20 +00:00
|
|
|
instance FromHttpApiData DataLength where
|
|
|
|
parseUrlPiece t = case readMaybe (T.unpack t) of
|
|
|
|
Nothing -> Left "X-git-annex-data-length parse error"
|
|
|
|
Just n -> Right (DataLength n)
|
|
|
|
|
|
|
|
instance ToJSON PutResult where
|
|
|
|
toJSON (PutResult b) =
|
|
|
|
object ["stored" .= b]
|
|
|
|
|
|
|
|
instance FromJSON PutResult where
|
|
|
|
parseJSON = withObject "PutResult" $ \v -> PutResult
|
|
|
|
<$> v .: "stored"
|
|
|
|
|
|
|
|
instance ToJSON PutResultPlus where
|
|
|
|
toJSON (PutResultPlus b us) = object
|
|
|
|
[ "stored" .= b
|
2024-07-08 01:20:50 +00:00
|
|
|
, "plusuuids" .= plusList us
|
2024-07-07 18:48:20 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
instance FromJSON PutResultPlus where
|
|
|
|
parseJSON = withObject "PutResultPlus" $ \v -> PutResultPlus
|
|
|
|
<$> v .: "stored"
|
|
|
|
<*> v .: "plusuuids"
|
|
|
|
|
|
|
|
instance ToJSON CheckPresentResult where
|
|
|
|
toJSON (CheckPresentResult b) = object
|
|
|
|
["present" .= b]
|
|
|
|
|
|
|
|
instance FromJSON CheckPresentResult where
|
|
|
|
parseJSON = withObject "CheckPresentResult" $ \v -> CheckPresentResult
|
|
|
|
<$> v .: "present"
|
|
|
|
|
|
|
|
instance ToJSON RemoveResult where
|
|
|
|
toJSON (RemoveResult b) = object
|
|
|
|
["removed" .= b]
|
|
|
|
|
|
|
|
instance FromJSON RemoveResult where
|
|
|
|
parseJSON = withObject "RemoveResult" $ \v -> RemoveResult
|
|
|
|
<$> v .: "removed"
|
|
|
|
|
|
|
|
instance ToJSON RemoveResultPlus where
|
|
|
|
toJSON (RemoveResultPlus b us) = object
|
|
|
|
[ "removed" .= b
|
2024-07-08 01:20:50 +00:00
|
|
|
, "plusuuids" .= plusList us
|
2024-07-07 18:48:20 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
instance FromJSON RemoveResultPlus where
|
|
|
|
parseJSON = withObject "RemoveResultPlus" $ \v -> RemoveResultPlus
|
|
|
|
<$> v .: "removed"
|
|
|
|
<*> v .: "plusuuids"
|
|
|
|
|
|
|
|
instance ToJSON GetTimestampResult where
|
|
|
|
toJSON (GetTimestampResult (Timestamp (MonotonicTimestamp t))) = object
|
|
|
|
["timestamp" .= t]
|
|
|
|
|
|
|
|
instance FromJSON GetTimestampResult where
|
|
|
|
parseJSON = withObject "GetTimestampResult" $ \v ->
|
|
|
|
GetTimestampResult . Timestamp . MonotonicTimestamp
|
|
|
|
<$> v .: "timestamp"
|
|
|
|
|
|
|
|
instance ToJSON PutOffsetResult where
|
|
|
|
toJSON (PutOffsetResult (Offset (P2P.Offset o))) = object
|
|
|
|
["offset" .= o]
|
|
|
|
|
|
|
|
instance FromJSON PutOffsetResult where
|
|
|
|
parseJSON = withObject "PutOffsetResult" $ \v ->
|
|
|
|
PutOffsetResult
|
|
|
|
<$> (Offset . P2P.Offset <$> v .: "offset")
|
|
|
|
|
|
|
|
instance ToJSON PutOffsetResultPlus where
|
|
|
|
toJSON (PutOffsetResultPlus (Offset (P2P.Offset o)) us) = object
|
|
|
|
[ "offset" .= o
|
2024-07-08 01:20:50 +00:00
|
|
|
, "plusuuids" .= plusList us
|
2024-07-07 18:48:20 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
instance FromJSON PutOffsetResultPlus where
|
|
|
|
parseJSON = withObject "PutOffsetResultPlus" $ \v ->
|
|
|
|
PutOffsetResultPlus
|
|
|
|
<$> (Offset . P2P.Offset <$> v .: "offset")
|
|
|
|
<*> v .: "plusuuids"
|
2024-07-08 01:20:50 +00:00
|
|
|
|
|
|
|
instance FromJSON (B64UUID t) where
|
|
|
|
parseJSON (String t) = case fromB64Maybe (TE.encodeUtf8 t) of
|
|
|
|
Just s -> pure (B64UUID (toUUID s))
|
|
|
|
_ -> mempty
|
|
|
|
parseJSON _ = mempty
|
|
|
|
|
convert lockcontent api to http long polling
Websockets would work, but the problem with using them for this is that
each lockcontent call is a separate websocket connection. And that's an
actual TCP connection. One TCP connection per file dropped would be too
expensive. With http long polling, regular http pipelining can be used,
so it will reuse a TCP connection.
Unfortunately, at least with servant, bi-directional streams with long
polling don't result in true bidirectional full duplex communication.
Servant processes the whole client body stream before generating the server
body stream. I think it's entirely possible to do full bi-directional
communication over http, but it would need changes to servant.
And, there's no way for the client to tell if the server successfully
locked the content, since the server will keep processing the client
stream no matter what.:
So, added a new api endpoint, keeplocked. lockcontent will lock the key
for 10 minutes with retention lock, and then a call to keeplocked will
keep it locked for as long as needed. This does mean that there will
need to be a Map of locks by key, and I will probably want to add
some kind of lock identifier that lockcontent returns.
2024-07-08 14:40:38 +00:00
|
|
|
instance ToJSON LockResult where
|
|
|
|
toJSON (LockResult v) = object
|
|
|
|
["locked" .= v]
|
|
|
|
|
|
|
|
instance FromJSON LockResult where
|
|
|
|
parseJSON = withObject "LockResult" $ \v -> LockResult
|
|
|
|
<$> v .: "locked"
|
|
|
|
|
|
|
|
instance ToJSON UnlockRequest where
|
|
|
|
toJSON (UnlockRequest v) = object
|
|
|
|
["unlock" .= v]
|
|
|
|
|
|
|
|
instance FromJSON UnlockRequest where
|
|
|
|
parseJSON = withObject "UnlockRequest" $ \v -> UnlockRequest
|
|
|
|
<$> v .: "unlock"
|
|
|
|
|
2024-07-08 01:20:50 +00:00
|
|
|
plusList :: [B64UUID Plus] -> [String]
|
|
|
|
plusList = map (\(B64UUID u) -> fromUUID u)
|
|
|
|
|
|
|
|
class PlusClass plus unplus where
|
|
|
|
dePlus :: plus -> unplus
|
|
|
|
plus :: unplus -> plus
|
|
|
|
|
|
|
|
instance PlusClass RemoveResultPlus RemoveResult where
|
|
|
|
dePlus (RemoveResultPlus b _) = RemoveResult b
|
|
|
|
plus (RemoveResult b) = RemoveResultPlus b mempty
|
|
|
|
|
|
|
|
instance PlusClass PutResultPlus PutResult where
|
|
|
|
dePlus (PutResultPlus b _) = PutResult b
|
|
|
|
plus (PutResult b) = PutResultPlus b mempty
|
|
|
|
|
|
|
|
instance PlusClass PutOffsetResultPlus PutOffsetResult where
|
|
|
|
dePlus (PutOffsetResultPlus o _) = PutOffsetResult o
|
|
|
|
plus (PutOffsetResult o) = PutOffsetResultPlus o mempty
|