Module:Sandbox/PHansen/URLutil

-- From de:Modul:URLutil -- Via en:User:PHansen/URLutil

-- Descriptions -- en: de:Wikipedia:Lua/Modul/URLutil/en -- de: de:Wikipedia:Lua/Modul/URLutil/de

-- Test : de:Wikipedia:Lua/Modul/URLutil/Test -- Wikidata: d:Q10859193

--[=[ URLutil 2014-09-20 Utilities for URL etc. on www. Only dotted decimal notation for IPv4 supported. Does not support dotted hexadecimal, dotted octal, or single-number formats. IPv6 URL (bracketed) not yet implemented; might need Wikintax escaping anyway. ]=]
 * getAuthority
 * getFragment
 * getHost
 * getLocation
 * getPath
 * getPort
 * getQuery
 * getQueryTable
 * getRelativePath
 * getScheme
 * getTLD
 * getTop2domain
 * getTop3domain
 * isAuthority
 * isDomain
 * isDomainExample
 * isDomainInt
 * isHost
 * isIP
 * isIPlocal
 * isIPv4
 * isIPv6
 * isMailAddress
 * isMailLink
 * isProtocolDialog
 * isProtocolWiki
 * isResourceURL
 * isSuspiciousURL
 * isUnescapedURL
 * isWebURL
 * wikiEscapeURL

-- table for export local URLutil = {}

URLutil.getURIScheme = function ( uri ) if type( uri ) == "string" then local prot, colon, slashes = uri:match( "^%s*([a-zA-Z]*)(:?)(/?/?)" ) if #colon == 1 and #prot >= 2 then return prot:lower elseif #slashes == 2 and #prot == 0 then return "//" end end return false end -- getURIScheme

local getTopDomain = function ( url, mode ) local r = URLutil.getHost( url ) if r then local pattern = "[%w%%]+%.%a[%w-]*%a)$"       if mode == 3 then            pattern = "[%w%%]+%." .. pattern        end        r = mw.ustring.match( "." .. r, "%.(" .. pattern ) if not r then r = false end else r = false end return r end -- getTopDomain

URLutil.getAuthority = function ( url ) local r   if type( url ) == "string" then local colon, host, port local pattern = "^%s*%w*:?//([%w%.%%-]+)(:?)([%d]*)/" local s = mw.text.decode( url ) local i = s:find( "#", 6, true ) if i then s = s:sub( 1, i - 1 )  .. "/"       else s = s .. "/"       end host, colon, port = mw.ustring.match( s, pattern ) if URLutil.isHost( host ) then host = mw.ustring.lower( host ) if colon == ":" then if port:find( "^[1-9]" ) then r = ( host .. ":" .. port ) end elseif #port == 0 then r = host end end else r = false end return r end -- URLutil.getAuthority

URLutil.getFragment = function ( url, decode ) local r   if type( url ) == "string" then local s = mw.text.decode( url ) local i = s:find( "#", 1, true ) if i then r = mw.text.trim( s:sub( i ) ):sub( 2 ) if type( decode ) == "string" then local encoding = mw.text.trim( decode ) local launch if encoding == "%" then launch = true elseif encoding == "WIKI" then r = r:gsub( "%.(%x%x)", "%%%1" ) :gsub( "_", " " ) launch = true end if launch then r = mw.uri.decode( r, "PATH" ) end end else r = false end else r = nil end return r end -- URLutil.getFragment

URLutil.getHost = function ( url ) local r = URLutil.getAuthority( url ) if r then r = mw.ustring.match( r, "^([%w%.%%-]+):?[%d]*$" ) end return r end -- URLutil.getHost

URLutil.getLocation = function ( url ) local r   if type( url ) == "string" then r = mw.text.trim( url ) if r == "" then r = false else local i           r = mw.text.decode( r ) i = r:find( "#", 1, true ) if i then if i == 1 then r = false else r = r:sub( 1, i - 1 ) end end end else r = nil end return r end -- URLutil.getLocation

URLutil.getPath = function ( url ) local r = URLutil.getRelativePath( url ) if r then local s = r:match( "^([^%?]*)%?" ) if s then r = s       end s = r:match( "^([^#]*)#" ) if s then r = s       end end return r end -- URLutil.getPath

URLutil.getPort = function ( url ) local r = URLutil.getAuthority( url ) if r then r = r:match( ":([1-9][0-9]*)$" ) if r then r = tonumber( r ) else r = false end end return r end -- URLutil.getPort

URLutil.getQuery = function ( url, key, separator ) local r = URLutil.getLocation( url ) if r then r = r:match( "^[^%?]*%?(.+)$" ) if r then if type( key ) == "string" then local single = mw.text.trim( key ) local sep = "&" local s, scan if type( separator ) == "string" then s = mw.text.trim( separator ) if s:match( "^[&;,/]$" ) then sep = s                   end end s = string.format( "%s%s%s", sep, r, sep ) scan = string.format( "%s%s=([^%s]*)%s",                                     sep, key, sep, sep ) r = s:match( scan ) end end if not r then r = false end end return r end -- URLutil.getQuery

URLutil.getQueryTable = function ( url, separator ) local r = URLutil.getQuery( url ) if r then local sep = "&" local n, pairs, s, set if type( separator ) == "string" then s = mw.text.trim( separator ) if s:match( "^[&;,/]$" ) then sep = s           end end pairs = mw.text.split( r, sep, true ) n = #pairs r = { } for i = 1, n do           s = pairs[ i ] if s:find( "=", 2, true ) then s, set = s:match( "^([^=]+)=(.*)$" ) if s then r[ s ] = set end else r[ s ] = false end end -- for i   end return r end -- URLutil.getQueryTable

URLutil.getRelativePath = function ( url ) local r   if type( url ) == "string" then local s = url:match( "^%s*[a-zA-Z]*://(.*)$" ) if s then s = s:match( "[^/]+(/.*)$" ) else local x           x, s = url:match( "^%s*(/?)(/.*)$" ) if x == "/" then s = s:match( "/[^/]+(/.*)$" ) end end if s then r = mw.text.trim( s ) elseif URLutil.isResourceURL( url ) then r = "/" else r = false end else r = nil end return r end -- URLutil.getRelativePath

URLutil.getScheme = function ( url ) local r   if type( url ) == "string" then local pattern = "^%s*([a-zA-Z]*)(:?)(//)" local prot, colon, slashes = url:match( pattern ) r = false if slashes == "//" then if colon == ":" then if #prot > 2 then r = prot:lower .. "://"               end elseif #prot == 0 then r = "//" end end else r = nil end return r end -- URLutil.getScheme

URLutil.getTLD = function ( url ) local r = URLutil.getHost( url ) if r then r = mw.ustring.match( r, "[%w]+%.(%a[%w-]*%a)$" ) if not r then r = false end end return r end -- URLutil.getTLD

URLutil.getTop2domain = function ( url ) return getTopDomain( url, 2 ) end -- URLutil.getTop2domain

URLutil.getTop3domain = function ( url ) return getTopDomain( url, 3 ) end -- URLutil.getTop3domain

URLutil.isAuthority = function ( s ) local r   if type( s ) == "string" then local pattern = "^%s*([%w%.%%-]+)(:?)(%d*)%s*$" local host, colon, port = mw.ustring.match( s, pattern ) if colon == ":" then port = port:match( "^[1-9][0-9]*$" ) if type( port ) ~= "string" then r = false end elseif port ~= "" then r = false end r = URLutil.isHost( host ) else r = nil end return r end -- URLutil.isAuthority

URLutil.isDomain = function ( s ) local r   if type( s ) == "string" then local scan = "^%s*([%w%.%%-]+%w)%.(%a[%w-]*%a)%s*$" local scope s, scope = mw.ustring.match( s, scan ) if type( s ) == "string" then if mw.ustring.find( s, "^%w" ) then if mw.ustring.find( s, "..", 1, true ) then r = false else r = true end end end else r = nil end return r end -- URLutil.isDomain

URLutil.isDomainExample = function ( url ) -- RFC 2606: example.com example.net example.org example.edu local r = getTopDomain( url, 2 ) if r then local s = r:lower:match( "^example%.([a-z][a-z][a-z])$" ) if s then r = ( s == "com" or                 s == "edu" or                  s == "net" or                  s == "org" ) else r = false end end return r end -- URLutil.isDomainExample

URLutil.isDomainInt = function ( url ) -- Internationalized Domain Name (Punycode) local r = URLutil.getHost( url ) if r then if r:match( "^[!-~]+$" ) then local s = "." .. r           if s:find( ".xn--", 1, true ) then r = true else r = false end else r = true end end return r end -- URLutil.isDomainInt

URLutil.isHost = function ( s ) return URLutil.isDomain( s ) or URLutil.isIP( s ) end -- URLutil.isHost

URLutil.isIP = function ( s ) return URLutil.isIPv4( s ) and 4 or URLutil.isIPv6( s ) and 6 end -- URLutil.isIP

URLutil.isIPlocal = function ( s ) -- IPv4 according to RFC 1918, RFC 1122; even any 0.0.0.0 (RFC 5735) local r = false local num = s:match( "^ *([01][0-9]*)%." ) if num then num = tonumber( num ) if num == 0 then r = s:match( "^ *0+%.[0-9]+%.[0-9]+%.[0-9]+ *$" ) elseif num == 10 or  num == 127 then -- loopback; private/local host: 127.0.0.1 r = URLutil.isIPv4( s ) elseif num == 169 then -- 169.254.*.*       elseif num == 172 then -- 172.(16...31).*.*           num = s:match( "^ *0*172%.([0-9]+)%." ) if num then num = tonumber( num ) if num >= 16 and  num <= 31 then r = URLutil.isIPv4( s ) end end elseif beg == 192 then -- 192.168.*.*           num = s:match( "^ *0*192%.([0-9]+)%." ) if num then num = tonumber( num ) if num == 168 then r = URLutil.isIPv4( s ) end end end end if r then r = true end return r end -- URLutil.isIPlocal

URLutil.isIPv4 = function ( s ) local function legal( n ) return ( tonumber( n ) < 256 ) end local r = false if type( s ) == "string" then local p1, p2, p3, p4 = s:match( "^%s*([1-9][0-9]?[0-9]?)%.([12]?[0-9]?[0-9])%.([12]?[0-9]?[0-9])%.([12]?[0-9]?[0-9])%s*$" ) if p1 and p2 and p3 and p4 then r = legal( p1 ) and legal( p2 ) and legal( p3 ) and legal( p4 ) end end return r end -- URLutil.isIPv4

URLutil.isIPv6 = function ( s ) local dcolon, groups if type( s ) ~= "string" or s:len == 0 or s:find( "[^:%x]" ) -- only colon and hex digits are legal chars or s:find( "^:[^:]" ) -- can begin or end with :: but not with single : or s:find( "[^:]:$" ) or s:find( ":::" ) then return false end s = mw.text.trim( s ) s, dcolon = s:gsub( "::", ":" ) if dcolon > 1 then return false end -- at most one :: s = s:gsub( "^:?", ":" ) -- prepend : if needed, upper s, groups = s:gsub( ":%x%x?%x?%x?", "" ) -- remove valid groups, and count them return ( ( dcolon == 1 and groups < 8 ) or            ( dcolon == 0 and groups == 8 ) ) and ( s:len == 0 or ( dcolon == 1 and s == ":" ) ) -- might be one dangling : if original ended with :: end -- URLutil.isIPv6

URLutil.isMailAddress = function ( s ) if type( s ) == "string" then s = mw.ustring.match( s, "^%s*[%w%.%%_-]+@([%w%.%%-]+)%s*$" ) return URLutil.isDomain( s ) end return false end -- URLutil.isMailAddress

URLutil.isMailLink = function ( s ) if type( s ) == "string" then local addr s, addr = mw.ustring.match( s, "^%s*([Mm][Aa][Ii][Ll][Tt][Oo]):(%S[%w%.%%_-]*@[%w%.%%-]+)%s*$" ) if type( s ) == "string" then if s:lower == "mailto" then return URLutil.isMailAddress( addr ) end end end return false end -- URLutil.isMailLink

local function isProtocolAccepted( prot, supplied ) if type( prot ) == "string" then local scheme, colon, slashes = mw.ustring.match( prot, "^%s*([a-zA-Z]*)(:?)(/?/?)%s*$" ) if slashes ~= "/" then if scheme == "" then if colon ~= ":" and slashes == "//" then return true end elseif colon == ":" or slashes == "" then local s = supplied:match( " " .. scheme:lower .. " " ) if type( s ) == "string" then return true end end end end return false end -- isProtocolAccepted

URLutil.isProtocolMW = function ( prot ) return isProtocolAccepted( prot,                              " http https ftp ftps ssh sftp irc ircs xmpp sip sips gopher telnet nntp worldwind mailto tel sms news svn git mms bitcoin magnet urn geo " ) end -- URLutil.isProtocolMW

URLutil.isProtocolDialog = function ( prot ) return isProtocolAccepted( prot, " mailto irc ircs ssh telnet " ) end -- URLutil.isProtocolDialog

URLutil.isProtocolWiki = function ( prot ) return isProtocolAccepted( prot,                              " ftp ftps git http https nntp sftp svn worldwind " ) end -- URLutil.isProtocolWiki

URLutil.isResourceURL = function ( url ) local scheme = URLutil.getScheme( url ) if scheme then local s = " // http:// https:// ftp:// sftp:// " s = s:find( string.format( " %s ", scheme ) ) if s then if URLutil.getAuthority( url ) then if not url:match( "%S%s+%S" ) then return true end end end end return false end -- URLutil.isResourceURL

URLutil.isSuspiciousURL = function ( url ) if URLutil.isResourceURL( url ) then local s = URLutil.getAuthority( url ) local pat = "[%[|%]" .. mw.ustring.char( 8201, 45, 8207, 8234, 45, 8239, 8288 ) .. "]"       if s:find( "@" ) or url:find( "''" ) or url:find( pat ) or url:find( "[%.,]$" ) then return true end -- TODO zero width character ?? return false end return true end -- URLutil.isSuspiciousURL

URLutil.isUnescapedURL = function ( url, trailing ) if type( trailing ) ~= "string" then if URLutil.isWebURL( url ) then if url:match( "[%[|%]]" ) then return true end end end return false end -- URLutil.isUnescapedURL

URLutil.isWebURL = function ( url ) if URLutil.getScheme( url ) and URLutil.getAuthority( url ) then if not url:match( "%S%s+%S" ) then return true end end return false end -- URLutil.isWebURL

URLutil.wikiEscapeURL = function ( url ) if url:find( "[%[|%]]" ) then local n       url, n = url:gsub( "%[", "&#91;" ) :gsub( "|", "&#124;" ) :gsub( "%]", "&#93;" ) end return url end -- URLutil.wikiEscapeURL

-- Provide template access and expose URLutil table to require

local p = {}

function p.getURIScheme( frame ) return URLutil.getURIScheme( frame.args[ 1 ] ) or "" end function p.getAuthority( frame ) return URLutil.getAuthority( frame.args[ 1 ] ) or "" end function p.getFragment( frame ) local r = URLutil.getFragment( frame.args[ 1 ], frame.args[ 2 ] ) if r then r = "#" .. r   else r = "" end return r end function p.getHost( frame ) return URLutil.getHost( frame.args[ 1 ] ) or "" end function p.getLocation( frame ) return URLutil.getLocation( frame.args[ 1 ] ) or "" end function p.getPath( frame ) return URLutil.getPath( frame.args[ 1 ] ) or "" end function p.getPort( frame ) return URLutil.getPort( frame.args[ 1 ] ) or "" end function p.getQuery( frame ) local r   local key = frame.args[ 2 ] if key then key = mw.text.trim( key ) if key == "" then key = nil end end r = URLutil.getQuery( frame.args[ 1 ], key, frame.args[ 3 ] ) if r then if not key then r = "?" .. r       end else r = "" end return r end function p.getRelativePath( frame ) return URLutil.getRelativePath( frame.args[ 1 ] ) or "" end function p.getScheme( frame ) return URLutil.getScheme( frame.args[ 1 ] ) or "" end function p.getTLD( frame ) return URLutil.getTLD( frame.args[ 1 ] ) or "" end function p.getTop2domain( frame ) return URLutil.getTop2domain( frame.args[ 1 ] ) or "" end function p.getTop3domain( frame ) return URLutil.getTop3domain( frame.args[ 1 ] ) or "" end function p.isAuthority( frame ) return URLutil.isAuthority( frame.args[ 1 ] ) and "1" or "" end function p.isDomain( frame ) return URLutil.isDomain( frame.args[ 1 ] ) and "1" or "" end function p.isDomainExample( frame ) return URLutil.isDomainExample( frame.args[ 1 ] ) and "1" or "" end function p.isDomainInt( frame ) return URLutil.isDomainInt( frame.args[ 1 ] ) and "1" or "" end function p.isHost( frame ) return URLutil.isHost( frame.args[ 1 ] ) and "1" or "" end function p.isIP( frame ) return URLutil.isIP( frame.args[ 1 ] ) or "" end function p.isIPlocal( frame ) return URLutil.isIPlocal( frame.args[ 1 ] ) and "1" or "" end function p.isIPv4( frame ) return URLutil.isIPv4( frame.args[ 1 ] ) and "1" or "" end function p.isIPv6( frame ) return URLutil.isIPv6( frame.args[ 1 ] ) and "1" or "" end function p.isMailAddress( frame ) return URLutil.isMailAddress( frame.args[ 1 ] ) and "1" or "" end function p.isMailLink( frame ) return URLutil.isMailLink( frame.args[ 1 ] ) and "1" or "" end function p.isProtocolMW( frame ) return URLutil.isProtocolMW( frame.args[ 1 ] ) and "1" or "" end function p.isProtocolDialog( frame ) return URLutil.isProtocolDialog( frame.args[ 1 ] ) and "1" or "" end function p.isProtocolWiki( frame ) return URLutil.isProtocolWiki( frame.args[ 1 ] ) and "1" or "" end function p.isResourceURL( frame ) return URLutil.isResourceURL( frame.args[ 1 ] ) and "1" or "" end function p.isSuspiciousURL( frame ) return URLutil.isSuspiciousURL( frame.args[ 1 ] ) and "1" or "" end function p.isUnescapedURL( frame ) return URLutil.isUnescapedURL( frame.args[ 1 ], frame.args[ 2 ] ) and "1" or "" end function p.isWebURL( frame ) return URLutil.isWebURL( frame.args[ 1 ] ) and "1" or "" end function p.wikiEscapeURL( frame ) return URLutil.wikiEscapeURL( frame.args[ 1 ] ) end function p.URLutil return URLutil end

return p