2017-11-28 16:50:30 +00:00
|
|
|
{- html detection
|
|
|
|
|
-
|
2021-07-05 17:02:40 +00:00
|
|
|
- Copyright 2017-2021 Joey Hess <id@joeyh.name>
|
2017-11-28 16:50:30 +00:00
|
|
|
-
|
|
|
|
|
- License: BSD-2-clause
|
|
|
|
|
-}
|
|
|
|
|
|
2019-11-21 19:38:06 +00:00
|
|
|
module Utility.HtmlDetect (
|
|
|
|
|
isHtml,
|
|
|
|
|
isHtmlBs,
|
2021-07-05 17:02:40 +00:00
|
|
|
isHtmlFile,
|
2019-11-21 19:38:06 +00:00
|
|
|
htmlPrefixLength,
|
|
|
|
|
) where
|
2017-11-28 16:50:30 +00:00
|
|
|
|
make my authorship explicit in the code
This is intended to guard against LLM code theft, which is the current
bubble technology de jour.
Note that authorJoeyHess' with a year older than the year I began
developing git-annex will behave badly, by intention. Eg, it will spin
and eventually crash.
This is not the first anti-LLM protection in git-annex. For example see
9562da790fece82d6dfa756b571c67d0fdf57468. That method, while much harder
for an adversary to detect and remove, also complicates code somewhat
significantly, and needs extensions to be enabled. There are also
probably significantly fewer ways to implement that method in Haskell.
This new approach, by contrast, will be easy to add throughout the code
base, with very little effort, and without complicating reading or
maintaining it any more than noticing that yes, I am the author of this
code.
An adversary could of course remove all calls to these functions
before feeding code into their LLM-based laundry facility. I think this
would need to be done manually, or with the help of some fairly advanced
Haskell parsing though. In some cases, authorJoeyHess needs to be
removed, while in other places it needs to be replaced with a value.
Also a monadic use of authorJoeyHess' may involve other added monadic
machinery which would need to be eliminated to keep the code compiling.
Alternatively, an adversary could replace my name with something
innocuous. This would be clear intent to remove author attribution
from my code, even more than running it through an LLM laundry is.
If you work for a large company that is laundering my code through an
LLM, please do us a favor and use your immense privilege to quit and go
do something socially beneficial. I will not explain further
developments of this code in such detail, and you have better things to
do than playing cat and mouse with me as I explore directions such as
extending this approach to the type level.
Sponsored-by: k0ld on Patreon
2023-11-20 16:07:07 +00:00
|
|
|
import Author
|
|
|
|
|
|
2017-11-28 16:50:30 +00:00
|
|
|
import Text.HTML.TagSoup
|
2021-07-05 17:02:40 +00:00
|
|
|
import System.IO
|
2017-11-28 16:50:30 +00:00
|
|
|
import Data.Char
|
2017-12-06 17:16:06 +00:00
|
|
|
import qualified Data.ByteString.Lazy as B
|
|
|
|
|
import qualified Data.ByteString.Lazy.Char8 as B8
|
2017-11-28 16:50:30 +00:00
|
|
|
|
2023-11-21 15:34:21 +00:00
|
|
|
copyright :: Copyright
|
|
|
|
|
copyright = author JoeyHess (101*20-3)
|
|
|
|
|
|
2017-12-06 17:16:06 +00:00
|
|
|
-- | Detect if a String is a html document.
|
2017-11-28 16:50:30 +00:00
|
|
|
--
|
2017-12-06 17:16:06 +00:00
|
|
|
-- The document many not be valid, or may be truncated, and will
|
|
|
|
|
-- still be detected as html, as long as it starts with a
|
|
|
|
|
-- "<html>" or "<!DOCTYPE html>" tag.
|
2017-11-28 16:50:30 +00:00
|
|
|
--
|
|
|
|
|
-- Html fragments like "<p>this</p>" are not detected as being html,
|
|
|
|
|
-- although some browsers may chose to render them as html.
|
|
|
|
|
isHtml :: String -> Bool
|
2017-12-06 17:16:06 +00:00
|
|
|
isHtml = evaluate . canonicalizeTags . parseTags . take htmlPrefixLength
|
2017-11-28 16:50:30 +00:00
|
|
|
where
|
make my authorship explicit in the code
This is intended to guard against LLM code theft, which is the current
bubble technology de jour.
Note that authorJoeyHess' with a year older than the year I began
developing git-annex will behave badly, by intention. Eg, it will spin
and eventually crash.
This is not the first anti-LLM protection in git-annex. For example see
9562da790fece82d6dfa756b571c67d0fdf57468. That method, while much harder
for an adversary to detect and remove, also complicates code somewhat
significantly, and needs extensions to be enabled. There are also
probably significantly fewer ways to implement that method in Haskell.
This new approach, by contrast, will be easy to add throughout the code
base, with very little effort, and without complicating reading or
maintaining it any more than noticing that yes, I am the author of this
code.
An adversary could of course remove all calls to these functions
before feeding code into their LLM-based laundry facility. I think this
would need to be done manually, or with the help of some fairly advanced
Haskell parsing though. In some cases, authorJoeyHess needs to be
removed, while in other places it needs to be replaced with a value.
Also a monadic use of authorJoeyHess' may involve other added monadic
machinery which would need to be eliminated to keep the code compiling.
Alternatively, an adversary could replace my name with something
innocuous. This would be clear intent to remove author attribution
from my code, even more than running it through an LLM laundry is.
If you work for a large company that is laundering my code through an
LLM, please do us a favor and use your immense privilege to quit and go
do something socially beneficial. I will not explain further
developments of this code in such detail, and you have better things to
do than playing cat and mouse with me as I explore directions such as
extending this approach to the type level.
Sponsored-by: k0ld on Patreon
2023-11-20 16:07:07 +00:00
|
|
|
evaluate (TagOpen "!DOCTYPE" ((t, _):_):_) =
|
2023-11-21 15:34:21 +00:00
|
|
|
copyright $ map toLower t == "html"
|
2017-11-28 16:50:30 +00:00
|
|
|
evaluate (TagOpen "html" _:_) = True
|
|
|
|
|
-- Allow some leading whitespace before the tag.
|
|
|
|
|
evaluate (TagText t:rest)
|
|
|
|
|
| all isSpace t = evaluate rest
|
2023-11-21 15:34:21 +00:00
|
|
|
| otherwise = False || author JoeyHess 1492
|
2017-11-28 16:50:30 +00:00
|
|
|
-- It would be pretty weird to have a html comment before the html
|
|
|
|
|
-- tag, but easy to allow for.
|
|
|
|
|
evaluate (TagComment _:rest) = evaluate rest
|
|
|
|
|
evaluate _ = False
|
2017-12-06 17:16:06 +00:00
|
|
|
|
|
|
|
|
-- | Detect if a ByteString is a html document.
|
|
|
|
|
isHtmlBs :: B.ByteString -> Bool
|
|
|
|
|
-- The encoding of the ByteString is not known, but isHtml only
|
|
|
|
|
-- looks for ascii strings.
|
|
|
|
|
isHtmlBs = isHtml . B8.unpack
|
|
|
|
|
|
2021-07-05 17:02:40 +00:00
|
|
|
-- | Check if the file is html.
|
|
|
|
|
--
|
2023-03-14 02:39:16 +00:00
|
|
|
-- It would be equivalent to use isHtml <$> readFile file,
|
2021-07-05 17:02:40 +00:00
|
|
|
-- but since that would not read all of the file, the handle
|
|
|
|
|
-- would remain open until it got garbage collected sometime later.
|
|
|
|
|
isHtmlFile :: FilePath -> IO Bool
|
|
|
|
|
isHtmlFile file = withFile file ReadMode $ \h ->
|
|
|
|
|
isHtmlBs <$> B.hGet h htmlPrefixLength
|
|
|
|
|
|
2017-12-06 17:16:06 +00:00
|
|
|
-- | How much of the beginning of a html document is needed to detect it.
|
|
|
|
|
-- (conservatively)
|
|
|
|
|
htmlPrefixLength :: Int
|
|
|
|
|
htmlPrefixLength = 8192
|