#!/usr/bin/env python
#copyright 2004 Mac-arena the Bored Zo.
#covered by the GNU General Public License (GPL) version 2 or later.
#if you did not receive a copy of the GPL: http://www.gnu.org/licenses/gpl.txt
import cgitb
cgitb.enable()
PATH_TO_QFLIB = '.'
import time
start_time = time.time()
import re, cgi, os, sys, random, textwrap, codecs, itertools, string
#print >>sys.stderr, 'Got a hit!'
from os import path
sys.path.insert(0, PATH_TO_QFLIB)
os.environ['QFLISTPATH'] = '/users/home/boredzo/.qfpaths'
import qflib
version = "0.3"
def strfromseq(seq):
return ', '.join(seq)
def strfromquotefile(qf, showlen=True):
if showlen:
return str(qf)
else:
s = HTMLescape(qf.filename)
return s
if showlen:
s += ' (%u entries)' % (len(qf.quotes),)
return s
def plural(s, n):
if n != 1: s += 's'
return s
def alleq(seq, n, num=int):
"""examines all the items of seq for equality with n. uses num to create the comparison number; by default, num is int.
equivalent to (not filter(lambda x: num(x) != n, seq)).
"""
for x in seq:
try:
i = num(x)
except ValueError:
return False
else:
if i != n: return False
else:
return True
"""
def listjoin(lists):
complete = []
for l in lists:
complete.extend(l)
return complete
"""
#output receptacles.
#presented here in order of preference.
plaintext = []
html = []
#entity-conversion magic.
entities = {
'<': '<',
'>': '>',
'&': '&',
'"': '"',
}
need_escape_exp = re.compile(u'[<>&"\x7f-\U00010FFF]')
def HTMLescape(instr):
unistr = codecs.utf_8_decode(instr)[0]
def get_entity(match, entities=entities):
match = match.group(0)
if match in entities:
return entities[match]
else:
return '&#x%x;' % (ord(match))
return codecs.utf_8_encode(need_escape_exp.sub(get_entity, instr))[0]
#this will be used for constructing forms.
class HTMLTag(object):
def __init__(self, _tag_name, _tag_name_transform = string.lower, _tag_contents = False, **_tag_attrs):
if callable(_tag_name_transform):
_tag_name = _tag_name_transform(_tag_name)
if not _tag_name: raise ValueError, "tag name must be a true string"
if not _tag_contents:
_tag_contents = "" #assure we get an empty string, not e.g. None
self.name = _tag_name
self.escaped_name = HTMLescape(_tag_name)
self.attrs = _tag_attrs
self.contents = _tag_contents
def __str__(self):
pieces = ['<', self.escaped_name] + [' %s=%r' % (name, HTMLescape(value)) for name, value in self.attrs.iteritems()] + ['>']
if self.contents:
pieces.append(HTMLescape(self.contents))
pieces.append('' % (self.escaped_name,))
return ''.join(pieces)
#text-wrapping magic.
width = default_wrap_width = 100
def createWrapper(width = default_wrap_width):
#TextWrapper didn't always work, so I crafted my own class to do the job.
#now I've got TextWrapper working, and TextWrapper is more robust, so this
# is an if True.
#eventually, MyWrapper and the if will go away.
if True:#False:
wrapper = textwrap.TextWrapper(
width = default_wrap_width,
expand_tabs = True,
replace_whitespace = False,
initial_indent = '',
subsequent_indent = ' '*4,
)
else:
class MyWrapper(object):
def wrap(self, s, width=80):
lines = s.splitlines()
result = []
for line in lines:
if line:
while len(line) > width:
i = width - 1
while (i >= 0) and not (line[i] in string.whitespace):
i -= 1
next_start = i + 1
while (i >= 0) and (line[i] in string.whitespace):
i -= 1
cur_end = i + 1
if cur_end < 1:
#long word
cur_end = next_start = width
result.append(line[:cur_end])
line = line[next_start:]
result.append(line)
return result
wrapper = MyWrapper()
class MultiLineWrapper(textwrap.TextWrapper):
def wraplines(self, input):
chunklists = [self.wrap(line) for line in input.splitlines()]
return list(itertools.chain(*chunklists))
wrapper = MultiLineWrapper(
width = default_wrap_width,
expand_tabs = True,
replace_whitespace = False,
initial_indent = '',
subsequent_indent = ' '*4,
)
def prepquote(instr, wrap=default_wrap_width):
if wrap > 0:
chunks = wrapper.wraplines(instr)
instr = '\n'.join(chunks)
return HTMLescape(instr)
#begin working on the CGI fields.
#used by 'date' and 'since' to parse dates.
#this is equivalent to 'YYYY-MM-DD'.
date_exp = re.compile('^([0-9]{4,})?[^0-9]([0-9]+)?[^0-9]([0-9]+)?$')
#directives (commands).
cmds = {}
def handle_str(form, key, qf):
strqueries = form[key]
strtitle = "String search results for " + strfromseq(strqueries)
titles[key] = strtitle
strtitle = section_header_fmt % (strtitle,)
body = []
func = "search_text"
if options.get('ignore_case', False):
func += '_insensitive'
results = {}
for text in strqueries:
matches = getattr(qf, func)(text)
if matches:
results[text] = matches
if results:
results_by_idx = {}
for l in results.itervalues():
for idx, entry in l:
results_by_idx[idx] = entry
indices = results_by_idx.keys()
indices.sort()
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for k in indices:
fmtargs = {
'linkURI': URI_fmt % (qfalias, k),
'index': k,
'quote': prepquote(qf.quotes[k]),
}
body.append(quote_fmt % fmtargs)
body.insert(0, strtitle)
else:
#no match
body.append(no_quote_fmt % ('plain-text', strfromquotefile(qf, showlen=False)))
return body
#FUTURE EXPANSION: handle_keywords (keyword searches)
def handle_regexp(form, key, qf):
re_queries = form[key]
re_title = "Regexp search results for " + strfromseq([x[0] for x in re_queries])
titles[key] = re_title
re_title = section_header_fmt % (re_title,)
body = []
results = {}
for text, exp in re_queries:
if options.get('ignore_case', False):
exp = re.compile(text, re.I)
matches = qf.search_re(exp)
if matches:
results[text] = matches
if results:
results_by_idx = {}
for l in results.itervalues():
for idx, entry, match in l:
results_by_idx[idx] = entry
indices = results_by_idx.keys()
indices.sort()
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for k in indices:
fmtargs = {
'linkURI': URI_fmt % (qfalias, k),
'index': k,
'quote': prepquote(qf.quotes[k]),
}
body.append(quote_fmt % fmtargs)
body.insert(0, re_title)
else:
#no match
body.append(no_quote_fmt % ('regexp', strfromquotefile(qf, showlen=False)))
return body
def handle_idx(form, key, qf):
idx_queries = form[key]
sg_range = options.get('shotgun', None)
if sg_range:
temp = dict(itertools.izip(idx_queries, itertools.repeat(None)))
if len(sg_range) == 1:
sg_range *= 2
for reqidx in idx_queries:
for idx in xrange(reqidx - sg_range[0], reqidx):
temp[idx] = None
reqidx += 1
for idx in xrange(reqidx, reqidx + sg_range[1]):
temp[idx] = None
idx_queries = temp.keys()
idx_queries.sort()
idx_title = "%u %s by index" % (len(idx_queries), plural('quote', len(idx_queries)))
titles[key] = idx_title
body = [section_header_fmt % (idx_title,)]
results = []
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for idx in idx_queries:
try:
quote = qf.quotes[idx]
except IndexError:
results.append("
No quote number %u
\n" % (idx,))
else:
fmtargs = {
'linkURI': URI_fmt % (qfalias, idx),
'index': idx,
'quote': prepquote(quote),
}
results.append(quote_fmt % fmtargs)
body.extend(results)
return body
def handle_date(form, key, qf):
datequeries = form[key]
date_titles = []
title_fmt = "%u %s on %u-%u-%u"
title_fmt_nohits = "No quotes on %u-%u-%u"
body = []
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for date in datequeries:
# print >> sys.stderr, 'Searching for date', repr(date)
results = qf.search_date(date)
#call qf.search_date on each quote-file, and print the results
if results:
title = (len(results), plural('quote', len(results))) + tuple(date)
date_titles.append(title_fmt % title)
body.append(section_header_fmt % (date_titles[-1],))
for idx, quote in results:
fmtargs = {
'linkURI': URI_fmt % (qfalias, idx),
'index': idx,
'quote': prepquote(quote),
}
body.append(quote_fmt % fmtargs)
else:
date_titles.append(title_fmt_nohits % tuple(date))
body.append(section_header_fmt % (date_titles[-1],))
titles[key] = title_separator.join(date_titles)
return body
#end handle_date
def handle_since(form, key, qf):
datequeries = form[key]
since_titles = []
title_fmt = "%u %s since %u-%u-%u"
title_fmt_nohits = "No quotes on or after %u-%u-%u"
body = []
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for date in datequeries:
fmtargs = {}
results = qf.search_date_or_after(date)
if results:
title = (len(results), plural('quote', len(results))) + tuple(date)
since_titles.append(title_fmt % title)
body.append(section_header_fmt % (since_titles[-1],))
for idx, quote in results:
fmtargs['linkURI'] = URI_fmt % (qfalias, idx)
fmtargs['index'] = idx
fmtargs['quote'] = prepquote(quote)
body.append(quote_fmt % fmtargs)
qflen = len(qf.quotes)
idx += 1
while idx < qflen:
fmtargs['linkURI'] = URI_fmt % (qfalias, idx)
fmtargs['index'] = idx
fmtargs['quote'] = prepquote(qf.quotes[idx])
body.append(quote_fmt % fmtargs)
idx += 1
else: #not results
since_titles.append(title_fmt_nohits % tuple(date))
body.append(section_header_fmt % (since_titles[-1],))
titles[key] = title_separator.join(since_titles)
return body
#end handle_since
def handle_first(form, key, qf):
firstqueries = form[key]
body = []
first_begin = 'First %u %s'
first_begin_1 = 'First quote'
first_end = "\n"#\n"
qfalias = qflib.lookup_filename(qf.filename)[0][0]
firstN = max(firstqueries)
if firstN:
if firstN == 1:
first_title = first_begin_1
else:
first_title = first_begin % (firstN, plural('quote', firstN))
titles[key] = first_title
body.append(section_header_fmt % (first_title,))
for i in xrange(firstN):
fmtargs = {
'linkURI': URI_fmt % (qfalias, i),
'index': i,
'quote': prepquote(qf.quotes[i]),
}
body.append(quote_fmt % fmtargs)
body.append(first_end)
return body
#end: handle_first
def handle_last(form, key, qf):
lastqueries = form[key]
body = []
last_begin = 'Last %u %s'
last_begin_1 = 'Last quote'
last_end = "\n"#\n"
qfalias = qflib.lookup_filename(qf.filename)[0][0]
lastN = max(lastqueries)
if lastN:
if lastN == 1:
last_title = last_begin_1
else:
last_title = last_begin % (lastN, plural('quote', lastN))
titles[key] = last_title
body.append(section_header_fmt % (last_title,))
max_idx = len(qf.quotes)
start = max_idx - lastN
if start < 0:
start = 0
while start < max_idx:
fmtargs = {
'linkURI': URI_fmt % (qfalias, start),
'index': start,
'quote': prepquote(qf.quotes[start]),
}
body.append(quote_fmt % fmtargs)
start += 1
body.append(last_end)
return body
#end: handle_last
def handle_random(form, key, qf):
randqueries = form[key]
body = []
#begin taking the random quotes.
rand_begin = '%u random %s'
rand_begin_1 = 'Random quote'
rand_end = "\n"#\n"
rand_titles = []
qfalias = qflib.lookup_filename(qf.filename)[0][0]
for randN in randqueries:
if randN == 1:
rand_titles.append(rand_begin_1)
else:
rand_titles.append(rand_begin % (randN, plural('quote', randN)))
body.append(section_header_fmt % (rand_titles[-1],))
anti_redundancy = {}
numquotes = len(qf.quotes)
while randN:
x = random.randrange(numquotes)
if x in anti_redundancy:
continue
else:
anti_redundancy[x] = True
randN -= 1
indices = anti_redundancy.keys()
indices.sort()
for i in indices:
fmtargs = {
'linkURI': URI_fmt % (qfalias, i),
'index': i,
'quote': prepquote(qf.quotes[i]),
}
body.append(quote_fmt % fmtargs)
body.append(rand_end)
titles[key] = title_separator.join(rand_titles)
return body
#end: handle_random
def handle_view(form, key, qf):
request = form[key]
# print >>sys.stderr, 'handle_view called; key =', repr(key), 'request =', repr(request)
if len(request) == 1 and request[0] in ('qf', 'qflib'):
bufsize = 8192
pathname = os.environ['SCRIPT_FILENAME']
if request[0] == 'qflib':
qflib_pathname = qflib.__file__
if path.splitext(qflib_pathname)[1] != '.py':
pathname = path.join(pathname, '..', 'qflib.py')
#and hope that's correct
#FIXME: in the future, iterate on sys.path and look for qflib
# that way, and be aware of the possibility that qflib.py may
# not exist (Python can work with only a .pyc).
else:
pathname = qflib_pathname
thisfile = file(pathname, 'r')
plaintext.append(thisfile.read())
thisfile.close()
return []
else:
body = []
for request in request:
if request == 'all':
qf.file.seek(0)
body.append(section_header_fmt % (HTMLescape(qf.strtitle),))
body.append('\n
')
else:
#request is the quote-file, *not* qf
request.file.seek(0)
plaintext.append(request.file.read())
else:
return body
#end: handle_view
cmd_handlers = {
'str': handle_str,
'regexp': handle_regexp,
'idx': handle_idx,
'date': handle_date,
'since': handle_since,
'first': handle_first,
'last': handle_last,
'random': handle_random,
'view': handle_view,
}
#pre-filters.
#call these with sets of arguments
# (e.g. for qf=sgf&qf=umf, you might call prefilter_qf(['sgf', 'umf'])).
def prefilter_qf(args):
if not args:
qflib.refreshqfs()
return qflib.knownqfs.keys()
else:
return args
prefilters = {
'qf': prefilter_qf
}
#filters.
#call these with an item from the form; they will return either a false value
# (False, (), None, etc) or something suitable for passing to the appropriate
# handler.
#note that because of filter_int_0, 0 (zero) is not a usable false value.
def filter_date(key, item): #date, since
match = date_exp.match(item)
if match:
try:
date = map(int, match.groups())
except ValueError:
# errors.append('%s: Dates must be YYYY-MM-DD, not %r' % (key, item))
return None
else:
return date
def filter_int(key, item): #first, last, random, wrap
try:
item = int(item)
except ValueError:
# errors.append('%s requires a number no less than 1, not %r' % (key, item))
return False
else:
if item > 0:
return item
else:
# errors.append('%s requires a number no less than 1, not %r' % (key, item))
return False
def filter_int_0(key, item): #idx
try:
item = int(item)
except ValueError:
# errors.append('%s requires a number no less than 0, not %r' % (key, item))
return False
else:
if item >= 0:
return item
else:
# errors.append('%s requires a number no less than 0, not %r' % (key, item))
return False
def filter_regexp(key, item): #regexp
try:
return (item, re.compile(item))
except re.error, err:
# errors.append('%s: re.compile(%r) says %r' % (key, item, err))
return None
def filter_view(key, item, constants=('qf', 'qflib', 'all', 'header', 'footer', 'quotes')): #view
# print 'filter_view:', repr(item)
if item in constants:
return item
else:
return filter_qf(key, item)
def filter_qf(key, item): #qf
qflib.refreshqfs()
# print >> sys.stderr, repr(item)
if item in qflib.knownqfs.iterkeys():
result = qflib.lookup(item)[0]
#this is too naive. it needs to be improved.
if result:
result = list(result)
result[0] = path.expanduser(result[0])
return qflib.UMF(*result)
else:
for info in qflib.knownqfs.itervalues():
for entry in info:
for pair in info:
if item == path.basename(pair[0]):
return qflib.UMF(path.expanduser(pair[0]))
elif item == path.expanduser(pair[0]):
return qflib.UMF(item[0])
elif item in pair:
return qflib.UMF(path.expanduser(pair[0]))
# errors.append('%s: %r is not a known quote-file' % (key, item))
# print >>sys.stderr, 'item not a known quote-file'
def filter_bool(key, item): #ignore_case
return item.lower() in ('true', 't', 'yes', 'y', '1')
filters = {
'date': filter_date, #cmd
'since': filter_date, #cmd
'idx': filter_int_0, #cmd
'first': filter_int, #cmd
'last': filter_int, #cmd
'random': filter_int, #cmd
'wrap': filter_int, #option
'shotgun': filter_int, #option
'regexp': filter_regexp, #cmd
'view': filter_view, #cmd
'qf': filter_qf, #option
'ignore_case': filter_bool, #option
}
#failure-message formats.
message_int = '“%(key)s” requires a number not less than one, not %(failed)r'
message_int_0 = '“%(key)s” requires a number not less than zero, not %(failed)r'
message_date = '“%(key)s” requires a YYYY-MM-DD date, not %(failed)r'
failure_fmts = {
'regexp': '%(failed)r is not a valid regular expression',
'view': '“%(key)s” requires “qf”, “qflib”, “header”, “quotes”, “footer”, or “all”, not %(failed)r',
'last': message_int,
'random': message_int,
'first': message_int,
'wrap': message_int,
'idx': message_int_0,
}
#options.
options = {}
def option_qf(item):
"open a quote-file."
def option_wrap(item):
global wrapper
# print >>sys.stderr, 'Creating wrapper to %u columns' % (item,)
wrapper.width = item
return wrapper.width
wrapper = MultiLineWrapper(
width = item,
expand_tabs = True,
replace_whitespace = False,
initial_indent = '',
subsequent_indent = ' '*4,
)
option_handlers = {
'wrap': option_wrap,#None,
'ignore_case': None,
'qf': None, #filter_qf,
'shotgun': None,
}
#final pre-flight.
form = cgi.FieldStorage()
# !!!DEPRECATED!!!
acceptable = ('qf',
#search parameters.
'date', 'since', 'str', 'regexp', 'idx', 'first', 'last', 'random', 'wrap', 'view',
#command can be view, random, or search. it is search by default.
#'command', #*DELETED*
#options.
#ignore_case can, of course, be either True or False.
'ignore_case',
#shotgun specifies the amount on either side of a given index to request.
#for example, with idx=5 and shotgun=2, the request is idx=range(3,8).
#can also have two numbers, specifying each side directly.
#for example, with idx=5 and shotgun=3,1, the request is idx=range(2,
# 7).
#may be extended to some other requests in the future (notably 'date').
'shotgun',)
# !!!DEPRECATED!!!
errors = []
titles = {}
title_separator = ', '
if __name__ == "__main__":
#make a dict out of 'form' so we can mutate it.
keys = form.keys()
newForm = {}
cmds = []
key_times = {}
for k in keys:
if k not in acceptable: continue
key_start_time = time.time()
newForm[k] = values = form.getlist(k)
if callable(prefilters.get(k, None)):
values[:] = prefilters[k](values)
if k in filters:
i = 0
while i < len(values):
new = filters[k](k, values[i])
# print >>sys.stderr, 'filter got', repr(values[i]), 'and returned', repr(new)
if new or (new is 0):
values[i] = new
i += 1
else:
if k in failure_fmts:
fmt_args = {
'key': k,
'failed': values[i],
}
errors.append(failure_fmts[k] % fmt_args)
del values[i]
if not values:
continue
if k in option_handlers:
# print >>sys.stderr, repr(k), option_handlers[k]
if callable(option_handlers[k]):
func = option_handlers[k]
new_values = [func(v) for v in values]
else:
new_values = list(values)
options[k] = new_values
# print >>sys.stderr, options[k]
if k in cmd_handlers:
cmds.append(k)
key_end_time = time.time()
if form.has_key(k):
key_times[k] = key_end_time - key_start_time
form = newForm; del newForm
# print >>sys.stderr, 'cmds:', cmds
# print >>sys.stderr, 'options:', options
if errors:
for i, err in enumerate(errors):
errors[i] = '
%s
\n' % (err,)
errors.insert(0, '
\n
Input errors
\n\n')
errors.append('
\n')
try:
quotefiles = options['qf']
except KeyError:
qflib.refreshqfs()
quotefiles = [qflib.UMF(*pair) for pair in itertools.chain(*qflib.knownqfs.itervalues())]
#get the URI pathname for the script, for use in quote links (see
# quote_fmt, above).
#for example: /irc/qf.py
#REQUEST_URI includes the query for a GET request, so we need to lop
# that off if present.
scriptURI = os.environ['REQUEST_URI']
query = os.environ.get('QUERY_STRING', '')
scriptURI = scriptURI[:-len(query)]
hasqmark = query.endswith('?')
scriptURI = scriptURI[:-hasqmark]
URI_fmt = scriptURI + '?qf=%s&idx=%u' #used in handlers for quote links
BAD_REQ = 'Status: 400 Bad Request'
HTML_wrapper_fmt = """
%s
Quote-File Central [BETA]
Version """ + version + """
%s
"""
time_taken_fmt = '\n
Time taken: %.3f seconds
'
if not cmds:
def menuForm():
"this doesn't do anything. it's for file-marking purposes."
pass
title = "Bored Zo Quote-File Central"
# body = []
section = \
"""
%(subheading)s
%(contents)s
"""
qf_checkbox_fmt = '
%s
'
qf_checkboxes = '\n'.join([qf_checkbox_fmt % (qflib.lookup_filename(qf.filename)[0][0], qf) for qf in quotefiles])
form = \
"""
"""
form_nosearch = """
"""
about = """
qf.py - web interface to qflib
version """ + version + """
copyright 2004 Mac-arena the Bored Zo (alias fGewA)