Monthly Archives: September 2011

MySQL to CSV

<pre id="comment_text_7">SELECT id,name,parent_id INTO OUTFILE 'r.csv'   FIELDS TERMINATED BY ','   LINES TERMINATED BY '\n'  FROM
wize2.categories;</pre>

To display all missing urls url_queue update command from merged mail

import time
import MySQLdb
conn = MySQLdb.connect (host = "10.241.31.96",
                           user = "root",
                           db = "spider")
cursor = conn.cursor ()

k=open("ms", "r")
a=[]

while(True):
        content = k.readline()
        if(content):

                if(content.find("url=http://www.")>0):
                        url=content[content.find("url=http://www.")+15:]

                        x=url[:url.find(",")]
                        if(len(x)>1):
                                x=x[:x.find(".")+4]
                                if(len(x)>1):
                                        x="www."+x
                                        a.append(x)

        else:
                break

a.sort()
while(True):
        x=a.pop()
        if(x in a):
                None

Fetching domains from email and inserting it into source table

<pre>import time
import MySQLdb
conn = MySQLdb.connect (host = "10.241.31.96",
                           user = "root",
                           db = "spider")
cursor = conn.cursor ()

k=open("ms", "r")
a=[]

while(True):
	content = k.readline()
	if(content):

		if(content.find("url=http://www.")>0):
			url=content[content.find("url=http://www.")+15:]


			x=url[:url.find(",")]
			if(len(x)>1):
				x=x[:x.find(".")+4]
				if(len(x)>1):
					x="www."+x
					a.append(x)
					print "=========================================="
	else:
		break


a.sort()
while(True):
	x=a.pop()
	if(x in a):
		None
	else:
		sql="""insert into sources(domain, min_rating, max_rating, is_disabled) values (%(x)s, 0, 5, 0)"""%dict(x=x)
		try:
		`	cursor.execute(sql)
		except Exception, ex:
			print ex
			break
		print "-------------"