Commit 2ab70be6 authored by bblanimation's avatar bblanimation

Major improvements to stability and usability

parent 96b27e69
......@@ -42,7 +42,7 @@ from bpy.props import *
# Render Farm imports
from .ui import *
from .buttons import *
from .functions.setupServers import *
from .functions import *
# Used to store keymaps for addon
addon_keymaps = []
......@@ -89,16 +89,10 @@ def register():
bpy.types.Scene.renderDumpLoc = StringProperty(
name="Output",
description="Folder to store output files from Blender (empty folder recommended)",
description="Output path for render files (empty folder recommended)",
maxlen=128,
default="//render-dump/",
subtype="DIR_PATH")
bpy.types.Scene.nameOutputFiles = StringProperty(
name="Name",
description="Name output files in 'render_dump' folder (prepended to: '_####')",
maxlen=128,
default="")
subtype="FILE_PATH")
bpy.types.Scene.maxServerLoad = IntProperty(
name="Max Server Load",
......@@ -195,7 +189,6 @@ def unregister():
del Scn.samplesPerFrame
del Scn.timeout
del Scn.maxServerLoad
del Scn.nameOutputFiles
del Scn.renderDumpLoc
del Scn.tempLocalDir
del Scn.frameRanges
......
......@@ -30,8 +30,6 @@ import time
from bpy.types import Operator
from bpy.props import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
class editRemoteServersDict(Operator):
"""Edit the remote servers dictionary in a text editor""" # blender will use this as a tooltip for menu items and buttons.
......
......@@ -30,8 +30,6 @@ import time
from bpy.types import Operator
from bpy.props import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
class listMissingFrames(Operator):
"""List the output files missing from the render dump folder""" # blender will use this as a tooltip for menu items and buttons.
......@@ -42,18 +40,12 @@ class listMissingFrames(Operator):
def execute(self, context):
try:
scn = context.scene
# initializes self.frameRangesDict (returns False if frame range invalid)
if not setFrameRangesDict(self):
return{"FINISHED"}
# list all missing files from start frame to end frame in render dump folder
missingFrames = listMissingFiles(getNameOutputFiles(), self.frameRangesDict["string"])
if len(missingFrames) > 0:
self.report({"INFO"}, "Missing frames: {missingFrames}".format(missingFrames=missingFrames))
else:
self.report({"INFO"}, "All frames accounted for!")
self.report({"INFO"}, "Missing frames: %(missingFrames)s" % locals() if len(missingFrames) > 0 else "All frames accounted for!")
return{"FINISHED"}
except:
handle_exception()
......@@ -69,14 +61,11 @@ class setToMissingFrames(Operator):
def execute(self, context):
try:
scn = context.scene
# initializes self.frameRangesDict (returns False if frame range invalid)
if not setFrameRangesDict(self):
return{"FINISHED"}
# list all missing files from start frame to end frame in render dump location
scn.frameRanges = listMissingFiles(getNameOutputFiles(), self.frameRangesDict["string"])
return{"FINISHED"}
except:
handle_exception()
......
......@@ -30,8 +30,6 @@ import time
from bpy.types import Operator
from bpy.props import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
class openRenderedImageInUI(Operator):
"""Open rendered image""" # blender will use this as a tooltip for menu items and buttons.
......@@ -71,19 +69,14 @@ class openRenderedAnimationInUI(Operator):
def execute(self, context):
scn = bpy.context.scene
try:
self.frameRangesDict = buildFrameRangesString(context.scene.frameRanges)
# change contexts
lastAreaType = changeContext(context, "CLIP_EDITOR")
# opens first frame of image sequence (blender imports full sequence)
openedFile = False
self.renderDumpFolder = getRenderDumpFolder()
image_sequence_filepath = "{dumpFolder}/".format(dumpFolder=self.renderDumpFolder)
for frame in bpy.props.animFrameRange:
image_filename = "{fileName}_{frame}{extension}".format(fileName=getNameOutputFiles(), frame=str(frame).zfill(4), extension=scn.animExtension)
if os.path.isfile(os.path.join(image_sequence_filepath, image_filename)):
bpy.ops.clip.open(directory=image_sequence_filepath, files=[{"name":image_filename}])
if os.path.isfile(os.path.join(self.renderDumpFolder, image_filename)):
bpy.ops.clip.open(directory=self.renderDumpFolder, files=[{"name":image_filename}])
openedFile = image_filename
openedFrame = frame
break
......@@ -92,9 +85,13 @@ class openRenderedAnimationInUI(Operator):
bpy.data.movieclips[openedFile].frame_start = frame
else:
changeContext(context, lastAreaType)
self.report({"ERROR"}, "Could not open rendered animation. View files in file browser in the following folder: '{renderDumpFolder}'.".format(renderDumpFolder=self.renderDumpFolder))
self.report({"ERROR"}, "Could not open rendered animation. View files in file browser in the following folder: '{path}'.".format(path=self.renderDumpFolder))
return{"FINISHED"}
except:
handle_exception()
return{"CANCELLED"}
def __init__(self):
self.frameRangesDict = buildFrameRangesString(context.scene.frameRanges)
self.renderDumpFolder = getRenderDumpPath()[0]
......@@ -30,9 +30,6 @@ import time
from bpy.types import Operator
from bpy.props import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
from ..functions.common import *
class refreshServers(Operator):
"""Attempt to connect to all servers through host server""" # blender will use this as a tooltip for menu items and buttons.
......@@ -130,7 +127,7 @@ class refreshServers(Operator):
return{"CANCELLED"}
@classmethod
def refreshServersBlock(cls, statusType=None):
def refreshServersBlock(cls):
scn = bpy.context.scene
if bpy.props.needsUpdating or bpy.props.lastServerGroup != scn.serverGroups:
bpy.props.lastServerGroup = scn.serverGroups
......
......@@ -31,8 +31,6 @@ from bpy.types import Operator
from bpy.props import *
from .refreshServers import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
class sendAnimation(Operator):
"""Render animation on remote servers""" # blender will use this as a tooltip for menu items and buttons.
......@@ -83,7 +81,7 @@ class sendAnimation(Operator):
for i in range(numIters):
self.processes[i].poll()
if self.processes[i].returncode != None:
if self.processes[i].returncode is not None:
# handle rsync error of no output files found on server
if self.state[i] == 4 and self.processes[i].returncode == 23:
if i == 1 and not self.statusChecked:
......@@ -111,13 +109,7 @@ class sendAnimation(Operator):
# handle unidentified errors
elif self.processes[i].returncode > 1:
setRenderStatus("animation", "ERROR")
# define self.errorSource string
if not self.state[i] == 3:
self.errorSource = "Processes[{i}] at state {state}".format(i=i, state=str(self.state[i]))
else:
self.errorSource = "blender_task"
self.errorSource = "Processes[{i}] at state {state}".format(i=i, state=str(self.state[i])) if self.state[i] != 3 else "blender_task"
handleError(self, self.errorSource, i)
setRenderStatus("animation", "ERROR")
self.cancel(context)
......@@ -156,10 +148,7 @@ class sendAnimation(Operator):
elif self.state[i] == 4:
numCompleted = getNumRenderedFiles("animation", self.expandedFrameRange, getNameOutputFiles())
if numCompleted > 0:
viewString = " - View rendered frames in render dump folder"
else:
viewString = ""
viewString = " - View rendered frames in render dump folder" if numCompleted > 0 else ""
self.report({"INFO"}, "Render completed for {numCompleted}/{numSent} frames{viewString}".format(numCompleted=numCompleted, numSent=len(bpy.props.animFrameRange), viewString=viewString))
scn.animPreviewAvailable = True
if i == 1:
......@@ -185,7 +174,6 @@ class sendAnimation(Operator):
def execute(self, context):
try:
self.projectName = bashSafeName(bpy.path.display_name_from_filepath(bpy.data.filepath))
scn = context.scene
# ensure no other render processes are running
......@@ -193,15 +181,11 @@ class sendAnimation(Operator):
self.report({"WARNING"}, "Render in progress...")
return{"CANCELLED"}
elif scn.availableServers == 0:
serversRefreshed = refreshServers.refreshServersBlock(statusType="animation")
serversRefreshed = refreshServers.refreshServersBlock()
if not serversRefreshed:
self.report({"WARNING"}, "Servers could not be auto-refreshed. Try manual refreshing (Ctrl R).")
return{"CANCELLED"}
# for testing purposes only (saves unsaved file)
if self.projectName == "":
self.projectName = "rf_unsaved_file"
print("\nRunning sendAnimation function...")
# ensure the job won't break the script
......@@ -220,19 +204,10 @@ class sendAnimation(Operator):
return{"CANCELLED"}
# set the file extension and frame range for use with 'open animation' button
scn.animExtension = bpy.context.scene.render.file_extension
scn.animExtension = scn.render.file_extension
bpy.props.animFrameRange = self.expandedFrameRange
# start initial render process
self.stdout = None
self.stderr = None
self.shift = False
self.renderCancelled = False
self.numFailedFrames = 0
self.startFrame = context.scene.frame_start
self.endFrame = context.scene.frame_end
self.numFrames = str(int(scn.frame_end) - int(scn.frame_start))
self.statusChecked = False
self.state = [1, 0] # initializes state for modal
if bpy.props.needsUpdating or bpy.props.lastServerGroup != scn.serverGroups:
bpy.props.lastServerGroup = scn.serverGroups
......@@ -262,6 +237,25 @@ class sendAnimation(Operator):
handle_exception()
return{"CANCELLED"}
def __init__(self):
scn = bpy.context.scene
# start initial render process
self.stdout = None
self.stderr = None
self.shift = False
self.renderCancelled = False
self.numFailedFrames = 0
self.startFrame = scn.frame_start
self.endFrame = scn.frame_end
self.numFrames = str(int(scn.frame_end) - int(scn.frame_start))
self.statusChecked = False
self.projectName = bashSafeName(bpy.path.display_name_from_filepath(bpy.data.filepath))
# for testing purposes only (saves unsaved file)
if self.projectName == "":
self.projectName = "rf_unsaved_file"
def cancel(self, context):
print("process cancelled")
cleanupCancelledRender(self, context, bpy.types.Scene.killPython)
......@@ -31,8 +31,6 @@ from bpy.types import Operator
from bpy.props import *
from .refreshServers import *
from ..functions import *
from ..functions.averageFrames import *
from ..functions.jobIsValid import *
class sendFrame(Operator):
"""Render current frame on remote servers""" # blender will use this as a tooltip for menu items and buttons.
......@@ -111,13 +109,7 @@ class sendFrame(Operator):
# handle unidentified errors
elif self.processes[i].returncode > 1:
setRenderStatus("image", "ERROR")
# define self.errorSource string
if not self.state[i] == 3:
self.errorSource = "Processes[{i}] at state {state}".format(i=i, state=str(self.state[i]))
else:
self.errorSource = "blender_task"
self.errorSource = "Processes[{i}] at state {state}".format(i=i, state=str(self.state[i])) if self.state[i] != 3 else "blender_task"
handleError(self, self.errorSource, i)
setRenderStatus("image", "ERROR")
self.cancel(context)
......@@ -216,9 +208,7 @@ class sendFrame(Operator):
def execute(self, context):
try:
self.projectName = bashSafeName(bpy.path.display_name_from_filepath(bpy.data.filepath))
scn = context.scene
if scn.render.engine != "CYCLES":
self.report({"INFO"}, "Rendering on local machine (switch to cycles to render current frame on remote servers).")
context.area.type = "IMAGE_EDITOR"
......@@ -231,39 +221,16 @@ class sendFrame(Operator):
self.report({"WARNING"}, "Render in progress...")
return{"CANCELLED"}
elif scn.availableServers == 0:
serversRefreshed = refreshServers.refreshServersBlock(statusType="image")
serversRefreshed = refreshServers.refreshServersBlock()
if not serversRefreshed:
self.report({"WARNING"}, "Servers could not be auto-refreshed. Try manual refreshing (Ctrl R).")
return{"CANCELLED"}
# for testing purposes only (saves unsaved file)
if self.projectName == "":
self.projectName = "rf_unsaved_file"
# ensure the job won't break the script
if not jobIsValid("image", self):
return{"CANCELLED"}
print("\nRunning sendFrame function...")
# set the file extension for use with 'open image' button
scn.imExtension = scn.render.file_extension
# Store current sample size for use in computing render results
self.sampleSize = scn.samplesPerFrame
# start initial render process
self.stdout = None
self.stderr = None
self.shift = False
self.renderCancelled = False
self.numSuccessFrames = 0
self.finishedFrames = 0
self.previewed = False
self.numSamples = 0
self.avDict = {"array":False, "numFrames":0}
self.averageIm = None
scn.imFrame = scn.frame_current
self.state = [1, 0] # initializes state for modal
if bpy.props.needsUpdating or bpy.props.lastServerGroup != scn.serverGroups:
bpy.props.lastServerGroup = scn.serverGroups
......@@ -294,6 +261,34 @@ class sendFrame(Operator):
handle_exception()
return{"CANCELLED"}
def __init__(self):
scn = bpy.context.scene
# set the file extension for use with 'open image' button
scn.imExtension = scn.render.file_extension
# Store current sample size for use in computing render results
self.sampleSize = scn.samplesPerFrame
# start initial render process
self.stdout = None
self.stderr = None
self.shift = False
self.renderCancelled = False
self.numSuccessFrames = 0
self.finishedFrames = 0
self.previewed = False
self.numSamples = 0
self.avDict = {"array":False, "numFrames":0}
self.averageIm = None
scn.imFrame = scn.frame_current
self.projectName = bashSafeName(bpy.path.display_name_from_filepath(bpy.data.filepath))
# for testing purposes only (saves unsaved file)
if self.projectName == "":
self.projectName = "rf_unsaved_file"
def cancel(self, context):
print("process cancelled")
cleanupCancelledRender(self, context, bpy.types.Scene.killPython)
This diff is collapsed.
......@@ -24,7 +24,7 @@ import bpy
import fnmatch
import numpy
import os
from . import getRenderDumpFolder
from .general import getRenderDumpPath
def averageFrames(classObject, outputFileName, verbose=0):
""" Averages final rendered images in blender to present one render result """
......@@ -32,16 +32,12 @@ def averageFrames(classObject, outputFileName, verbose=0):
if verbose >= 1:
print("Averaging images...")
# ensure renderedFramesPath has trailing "/"
renderedFramesPath = getRenderDumpFolder()
if not renderedFramesPath.endswith("/"):
renderedFramesPath += "/"
# get image files to average from 'renderedFramesPath'
allFiles = os.listdir(renderedFramesPath)
# get image files to average
renderPath = getRenderDumpPath()[0]
allFiles = os.listdir(renderPath)
inFileName = "{outputFileName}_seed-*_{frame}{extension}".format(outputFileName=outputFileName, frame=str(scn.imFrame).zfill(4), extension=scn.imExtension)
imListNames = [filename for filename in allFiles if fnmatch.fnmatch(filename, inFileName)]
imList = [os.path.join(renderedFramesPath, im) for im in imListNames]
imList = [os.path.join(renderPath, im) for im in imListNames]
if not imList:
print("No image files to average")
return None
......
This diff is collapsed.
......@@ -21,7 +21,7 @@ Created by Christopher Gearhart
# system imports
import bpy
from . import getRenderDumpFolder
from .general import getRenderDumpPath
def jobIsValid(jobType, classObject):
""" verifies that the job is valid before sending it to the host server """
......@@ -54,10 +54,9 @@ def jobIsValid(jobType, classObject):
jobValidityDict = {"valid":False, "errorType":"WARNING", "errorMessage": "Max Samples / SamplesPerJob > 100. Try increasing samples per frame or lowering max samples."}
# verify that the user input for renderDumpLoc is valid and can be created
try:
rdf = getRenderDumpFolder()
except:
jobValidityDict = {"valid":False, "errorType":"ERROR", "errorMessage":"The folder '{renderDumpFolder}' could not be created on your local machine. Verify your input and write permissions or try another filepath.".format(renderDumpFolder=scn.renderDumpLoc)}
rdf, errorMsg = getRenderDumpPath()
if errorMsg is not None:
jobValidityDict = {"valid":False, "errorType":"ERROR", "errorMessage":errorMsg}
# else, the job is valid
if not jobValidityDict:
......
......@@ -78,11 +78,17 @@ matt
### BEGIN REMOTE SERVERS DICTIONARY ###
{'cse217': [
"cse21701","cse21702","cse21703","cse21704","cse21705","cse21706",
"cse21707","cse21708","cse21709","cse21710","cse21711","cse21712",
"cse21707","cse21709","cse21710","cse21712",
"cse21713","cse21714","cse21715","cse21716"
],'cse218': [
"cse21801","cse21802", "cse21803","cse21804","cse21805","cse21806",
"cse21807","cse21808","cse21809","cse21810","cse21811","cse21812"
"cse21807","cse21808","cse21809","cse21811","cse21812",
"cse21814"
],'cse201': [
"cse20101","cse20102","cse20103","cse20104","cse20105","cse20106",
"cse20107","cse20108","cse20109","cse20110","cse20111","cse20112",
"cse20113","cse20114","cse20115","cse20116","cse20117","cse20118",
"cse20119","cse20120","cse20121","cse20122","cse20123","cse20124"
],'cse103': [
"cse10301","cse10302","cse10303","cse10304","cse10305","cse10306",
"cse10307","cse10308","cse10309","cse10310","cse10311","cse10312",
......@@ -91,9 +97,26 @@ matt
]}
### END REMOTE SERVERS DICTIONARY ###
# 201 servers
### NOTES (everything below this line will be ignored) ###
1: 4-6.5
2: 6.5-10
3: 10-14
217 in order:
"cse21701","cse21702","cse21703","cse21705","cse21707","cse21710","cse21712","cse21713","cse21715"
"cse21706","cse21716",
"cse21708","cse21711",
unchecked: 4, 9, 14
218 in order:
unchecked: 1,2,3,4,5,6,7,8,9,10,11,12,13,14
],'cse201': [
"cse20101","cse20102","cse20103","cse20104","cse20105","cse20106",
"cse20107","cse20108","cse20109","cse20110","cse20111","cse20112",
"cse20113","cse20114","cse20115","cse20116","cse20117","cse20118",
"cse20119","cse20120","cse20121","cse20122","cse20123","cse20124"
"cse20119","cse20120","cse20121","cse20122","cse20123","cse20124",
"cse21810","cse21813","cse21814"
\ No newline at end of file
......@@ -121,18 +121,10 @@ class JobHost(threading.Thread):
while True:
if acc < self.max_on_host:
job = self.get_next_job()
if job:
# Start jobs in the pool if we are not already past the max running jobs
if job not in self.jobs:
self.started = True
self.jobs[job] = dict()
self.kwargs["jobString"] = job
self.kwargs["hostname"] = self.get_hostname()
self.kwargs["firstTime"] = self.firstTime
job_process=Process(target=self.thread_func,kwargs=self.kwargs)
job_process.start()
self.jobs[job]['process'] = job_process
acc += 1
# Start jobs in the pool if we are not already past the max running jobs
if job and job not in self.jobs:
self.start_job(job)
acc += 1
# Check on child processes
for job_key in self.jobs.keys():
job_process=self.jobs[job_key]['process']
......@@ -177,7 +169,7 @@ class JobHost(threading.Thread):
return (self.job_count <= self.max_on_host) and (len(self.jobs_list) > 0)
def can_take_job(self):
return self.job_count <= self.max_on_host
return self.job_count < self.max_on_host
def get_callback(self):
return self.callback
......@@ -185,6 +177,16 @@ class JobHost(threading.Thread):
def get_error_callback(self):
return self.error_callback
def start_job(self, job):
self.started = True
self.jobs[job] = dict()
self.kwargs["jobString"] = job
self.kwargs["hostname"] = self.get_hostname()
self.kwargs["firstTime"] = self.firstTime
job_process=Process(target=self.thread_func,kwargs=self.kwargs)
job_process.start()
self.jobs[job]['process'] = job_process
def job_complete(self, job=None, exit_status=0):
self.firstTime = False
self.job_count -= 1
......
......@@ -36,6 +36,7 @@ class JobHostManager():
self.hosts = dict()
if not hosts: self.hosts = dict()
else: self.add_hosts(hosts)
self.host_keys = sorted(self.hosts.keys(), reverse=True)
self.hosts_with_jobs = dict()
self.job_status = dict()
......@@ -55,11 +56,11 @@ class JobHostManager():
try:
while not self.jobs_complete() and not self.stop_now:
# time.sleep(.25)
for aHost in self.hosts.keys():
for aHost in self.host_keys:
if self.jobs_complete() or self.stop_now:
break
host = self.hosts[aHost]
while self.host_can_take_job(host=host) and len(self.jobs) > 0:
if self.host_can_take_job(host=host) and len(self.jobs) > 0:
# time.sleep(.1)
hostname = host.get_hostname()
job = self.jobs.pop()
......@@ -73,6 +74,9 @@ class JobHostManager():
pflush("Running job {job} on host {host}. ({numQueued} jobs remain in queue)".format(job=job, host=hostname, numQueued=numQueued))
elif self.verbose >= 2:
pflush("Job sent to host '{hostname}' ({numQueued} jobs remain in queue)".format(hostname=hostname, numQueued=numQueued))
# else:
# pflush("Job sent to host '{hostname}' ({numRemaining} jobs left)".format(hostname=hostname, numRemaining=self.remaining_jobs()))
pflush("Job sent to host '{hostname}' ({numQueued} jobs remain in queue)".format(hostname=hostname, numQueued=numQueued))
self.stop_all_threads()
except (KeyboardInterrupt, SystemExit):
self.stop_all_threads()
......
......@@ -24,8 +24,7 @@ import bpy
import math
from bpy.types import Panel
from bpy.props import *
from ..functions import getRenderStatus, have_internet
from ..functions.setupServers import *
from ..functions import *
from .app_handlers import *
class renderOnServersPanel(Panel):
......@@ -139,8 +138,8 @@ class serversPanel(Panel):
box.prop(scn, "showAdvanced")
if scn.showAdvanced:
col = box.column()
col.prop(scn, "nameOutputFiles")
col.prop(scn, "renderDumpLoc")
col.label("Output:")
col.prop(scn, "renderDumpLoc", text="")
layout.separator()
......
......@@ -23,20 +23,22 @@ Created by Christopher Gearhart
import bpy
import math
from bpy.app.handlers import persistent
from bpy.types import Panel
from bpy.props import *
from ..buttons.refreshServers import refreshServers
from ..functions import *
from ..functions.setupServers import *
@persistent
def refresh_servers(scene):
updateServerPrefs()
bpy.app.handlers.load_post.append(refresh_servers)
@persistent
def verify_render_status_on_load(scene):
scn = bpy.context.scene
if scn.imageRenderStatus in ["Preparing files...", "Rendering...", "Finishing..."]:
scn.imageRenderStatus = "None"
if scn.animRenderStatus in ["Preparing files...", "Rendering...", "Finishing..."]:
scn.animRenderStatus = "None"
scn.imageRenderStatus = "None" if scn.imageRenderStatus in ["Preparing files...", "Rendering...", "Finishing...", "ERROR"] else scn.imageRenderStatus
scn.animRenderStatus = "None" if scn.animRenderStatus in ["Preparing files...", "Rendering...", "Finishing...", "ERROR"] else scn.animRenderStatus
bpy.app.handlers.load_post.append(verify_render_status_on_load)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment