I am new to the python language, so please bear with me. Also English isn’t my native language so sorry for any misspelled words.
I have a question about updating a Django app from a daemon that runs locally on my server. I have a server setup which has 8 hot-swappable bays. Users can plug-in there hard disk(s) into the server and, after the server has detected that a new hard disk is plugged-in, it starts copying the contents of the hard disk to a location on the network. The current setup displays information about the process on an LCD screen.
The current setup works fine but I need to change it in a way that the whole process is displayed on a website (since this is more user friendly). So I need to display to the user when a disk is inserted into the server, the progress of the copy task etc.
My idea it to create a Django app that gets updated when a task in process is completed, but I can’t seem to find any information about updating a Django app from a locally running daemon. It this even possible? Or is Django not the right way to go? Any ideas are welcome.
Below is my script used to copy content of disk to a location on the network. Hopefully it give some more information about what I doing/tying to do.
Many thanks in advance!
Script:
#!/usr/bin/env python
import os
import sys
import glob
import re
import time
import datetime
import pyudev
import thread
import Queue
import gobject
import getopt
from pyudev import Context
from subprocess import Popen, PIPE
from subprocess import check_call
from lcdproc.server import Server
from pyudev.glib import GUDevMonitorObserver
from gobject import MainLoop
from threading import Thread
#used to show progress info
from progressbar import ProgressBar, Percentage, Bar, RotatingMarker, ETA, FileTransferSpeed
# used to set up screens
lcd = Server("localhost", 13666, debug=False)
screens = []
widgets = []
#Used for threading
disk_work_queue = Queue.Queue()
# used to store remote nfs folders
remote_dirs = ['/mnt/nfs/', '/mnt/nfs1/', '/mnt/nfs2/']
#Foldername on remote server (NFS Share name)
REMOTE_NFS_SHARE = ''
# a process that runs infinity, it starts disk processing
# functions.
class ProcessThread(Thread):
def __init__(self):
Thread.__init__(self)
def run(self):
while 1:
try:
disk_to_be_processed = disk_work_queue.get(block=False)
set_widget_text(disk_to_be_processed[1], "Removed from queue..", "info", "on")
process_disk(disk_to_be_processed[0], disk_to_be_processed[1])
except Queue.Empty:
time.sleep(10)
set_main_widget_text("Please insert disks ")
# used to set message on the lcdscreen, message are set by disk
def set_widget_text(host, message, priority, blacklight):
if host == "host4":
screen_disk1 = screens[1]
screen_disk1.clear()
screen_disk1.set_priority(priority)
screen_disk1.set_backlight(blacklight)
widgets[1].set_text(str(message))
elif host == "host5":
screen_disk2 = screens[2]
screen_disk2.clear()
screen_disk2.set_priority(priority)
screen_disk2.set_backlight(blacklight)
widgets[2].set_text(str(message))
elif host == "host6":
screen_disk3 = screens[3]
screen_disk3.clear()
screen_disk3.set_priority(priority)
screen_disk3.set_backlight(blacklight)
widgets[3].set_text(str(message))
elif host == "host7":
screen_disk4 = screens[4]
screen_disk4.clear()
screen_disk4.set_priority(priority)
screen_disk4.set_backlight(blacklight)
widgets[4].set_text(str(message))
# used to set a message for all hosts
def set_widget_text_all(hosts, message, priority, blacklight):
for host in hosts:
set_widget_text(host, message, priority, blacklight)
def set_main_widget_text(message):
screen_disk1 = screens[0]
screen_disk1.clear()
screen_disk1.set_priority("info")
screen_disk1.set_backlight("on")
widgets[0].set_text(str(message))
# mounts, find logs files and copy image files to destionation
def process_disk(disk, host):
datadisk = mount_disk(disk, host)
source = datadisk + "/images"
set_widget_text(host, "Processing, hold on ", "info", "on")
cases = find_log(source)
upload(source, cases, host)
time.sleep(5)
umount_disk(host)
set_widget_text(host, "Disk can be removed", "info", "blink")
time.sleep(10)
# search the datadisk for logfiles containing information
# about cases and images
def find_log(src):
inf = ""
case = []
for root,dirs,files in os.walk(src):
for f in files:
if f.endswith(".log"):
log = open(os.path.join(root,f), 'r')
lines = log.readlines()[2:5]
for l in lines:
inf += re.sub("\n","",l[11:]) + ":"
log.close()
print inf
case.append(inf)
inf = ""
return case
def get_directory_size(dir):
dir_size = 0
for(path, dirs, files) in os.walk(dir):
for file in files:
filename = os.path.join(path, file)
dir_size+=os.path.getsize(filename)
return dir_size
# copies the image files to the destination location, dc3dd is used
# to copy the files in a forensicly correct way.
def upload(src, cases, host):
remotedir = ''
while len(cases) > 0:
count = 0
nfs_share_found = False
case = cases.pop()
onderzoek = case.split(':')[0];
#verwijder de _ uit de naam van het object
object = case.split(':')[1];
#image = case.split(':')[2];
localdir = src + '/' + onderzoek + '/' + object +'/'
total_files = len(os.listdir(localdir))
folder_size = get_directory_size(localdir)
for d in remote_dirs:
if os.path.exists(d + onderzoek + '/B/' + object.replace('_',' ') + '/Images/'):
nfs_share_found = True
remotedir = d + onderzoek + '/B/' + object.replace('_', ' ') + '/Images/'
break
if nfs_share_found == False:
set_widget_text(host, " Onderzoek onbekend ", "info", "flash")
time.sleep(30)
return
for root,dirs,files in os.walk(localdir):
for uploadfile in files:
currentfile = os.path.join(root, uploadfile)
file_size = os.stat(currentfile).st_size
copy_imagefile(currentfile, onderzoek, object, remotedir)
count += 1
percentage = int(count*file_size*100/folder_size)
message = onderzoek + " Obj: " + object + "..%d%%" % percentage
set_widget_text(host, message, "info", "on")
set_widget_text(host, " Copy Succesfull! ", "info", "flash")
# the actualy function to copy the files, using dc3dd
def copy_imagefile(currentfile, onderzoek, object, remotedir):
currentfilename = os.path.basename(currentfile)
dc3dd = Popen(["dc3dd", "if=" + currentfile, "hash=md5", "log=/tmp/"+ onderzoek + "_" + object + ".log", "hof=" + remotedir + currentfilename,"verb=on", "nwspc=on"],stdin=PIPE,stdout=PIPE, stderr=PIPE)
dc3dd_stdout = dc3dd.communicate()[1]
awk = Popen([r"awk", "NR==13 { print $1 }"],stdin=PIPE, stdout=PIPE)
awk_stdin = awk.communicate(dc3dd_stdout)[0]
output = awk_stdin.rstrip('\n')
if output == "[ok]":
return False
else:
return True
# when a disk gets inserted into the machine this function is called to prepare the disk
# for later use.
def device_added_callback(self, device):
position = device.sys_path.find('host')
host = device.sys_path[(position):(position+5)]
set_widget_text(host, " New disk inserted! ", "info", "on")
time.sleep(2)
disk = "/dev/" + device.sys_path[-3:] + "1"
disk_work_queue.put((disk, host))
set_widget_text(host, " Placed in queue... ", "info", "on")
# gets called when the disk is removed form the machine
def device_removed_callback(self, device):
position = device.sys_path.find('host')
host = device.sys_path[(position):(position+5)]
#message = 'Slot %s : Please remove drive' % host[4:]
set_widget_text(host, " Replace disk ", "info", "on")
# mounts the partition on the datadisk
def mount_disk(disk, host):
#device = "/dev/" + disk + "1"
mount_point = "/mnt/" + host
if not os.path.exists(mount_point):
os.mkdir(mount_point)
cmd = ['mount', '-o', 'ro,noexec,noatime,nosuid', str(disk), str(mount_point)]
check_call(cmd)
set_widget_text(host, " Disk mounted ", "info", "on")
return mount_point
# umounts the partition datadisk
def umount_disk(host):
mount_point = "/mnt/" + host
cmd = ['umount', str(mount_point)]
check_call(cmd)
os.removedirs(mount_point)
def build_screens():
screen_main = lcd.add_screen("MAIN")
screen_main.set_heartbeat("off")
screen_main.set_duration(3)
screen_main.set_priority("background")
widget0_1 = screen_main.add_string_widget("screen0Widget1", " Welcome to AFFC ", x=1, y=1)
widget0_2 = screen_main.add_string_widget("screen0Widget2", "Please insert disks ", x=1, y=2)
widgets.append(widget0_2)
screens.append(screen_main)
screen_disk1 = lcd.add_screen("DISK1")
screen_disk1.set_heartbeat("off")
screen_disk1.set_duration(3)
screen_disk1.clear()
widget_disk1_1 = screen_disk1.add_string_widget("disk1Widget1", " Slot 1 ", x=1, y=1)
widget_disk1_2 = screen_disk1.add_string_widget("disk1Widget2", " Please insert disk ", x=1, y=2)
widgets.append(widget_disk1_2)
screens.append(screen_disk1)
screen_disk2 = lcd.add_screen("DISK2")
screen_disk2.set_heartbeat("off")
screen_disk2.set_duration(3)
widget_disk2_1 = screen_disk2.add_string_widget("disk2Widget1", " Slot 2 ", x=1, y=1)
widget_disk2_2 = screen_disk2.add_string_widget("disk2Widget2", " Please insert disk ", x=1, y=2)
widgets.append(widget_disk2_2)
screens.append(screen_disk2)
screen_disk3 = lcd.add_screen("DISK3")
screen_disk3.set_heartbeat("off")
screen_disk3.set_duration(3)
widget_disk3_1 = screen_disk3.add_string_widget("disk3Widget1", " Slot 3 ", x=1, y=1)
widget_disk3_2 = screen_disk3.add_string_widget("disk3Widget2", " Please insert disk ", x=1, y=2)
widgets.append(widget_disk3_2)
screens.append(screen_disk3)
screen_disk4 = lcd.add_screen("DISK4")
screen_disk4.set_heartbeat("off")
screen_disk4.set_duration(3)
widget_disk4_1 = screen_disk4.add_string_widget("disk4Widget1", " Slot 4 ", x=1, y=1)
widget_disk4_2 = screen_disk4.add_string_widget("disk4Widget2", " Please insert disk ", x=1, y=2)
widgets.append(widget_disk4_2)
screens.append(screen_disk4)
def restart_program():
"""Restarts the current program.
Note: this function does not return. Any cleanup action (like
saving data) must be done before calling this function."""
python = sys.executable
os.execl(python, python, * sys.argv)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "hd:v", ["help", "destination="])
except getopt.GetoptError, err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
verbose = False
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-d", "--destination"):
REMOTE_NFS_SHARE = a
else:
assert False, "unhandled option"
lcd.start_session()
build_screens()
#t = Thread(target=loop_disks_process())
#t.start();
context = pyudev.Context()
monitor = pyudev.Monitor.from_netlink(context)
observer = GUDevMonitorObserver(monitor)
observer.connect('device-added', device_added_callback)
observer.connect('device-removed', device_removed_callback)
monitor.filter_by(subsystem='block', device_type='disk')
monitor.enable_receiving()
mainloop = MainLoop()
gobject.threads_init()
t = ProcessThread()
t.start()
mainloop.run()
raw_input("Hit <enter>")
t.running = False
t.join()
if __name__ == "__main__":
try:
main()
except Exception, e:
restart_program()
Take a look at Django Piston. You can implement a RESTful API on your django app and call those apis from your demon. I use it on one of my project in which some worker processes need to communicate to frontend django apps periodically.