jobs.py 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183
  1. # -*- coding: utf-8 -*-
  2. import os
  3. import sys
  4. from imp import find_module
  5. from typing import Optional # NOQA
  6. from django.apps import apps
  7. _jobs = None
  8. def noneimplementation(meth):
  9. return None
  10. class JobError(Exception):
  11. pass
  12. class BaseJob:
  13. help = "undefined job description."
  14. when = None # type: Optional[str]
  15. def execute(self):
  16. raise NotImplementedError("Job needs to implement the execute method")
  17. class MinutelyJob(BaseJob):
  18. when = "minutely"
  19. class QuarterHourlyJob(BaseJob):
  20. when = "quarter_hourly"
  21. class HourlyJob(BaseJob):
  22. when = "hourly"
  23. class DailyJob(BaseJob):
  24. when = "daily"
  25. class WeeklyJob(BaseJob):
  26. when = "weekly"
  27. class MonthlyJob(BaseJob):
  28. when = "monthly"
  29. class YearlyJob(BaseJob):
  30. when = "yearly"
  31. def my_import(name):
  32. try:
  33. imp = __import__(name)
  34. except ImportError as err:
  35. raise JobError("Failed to import %s with error %s" % (name, err))
  36. mods = name.split('.')
  37. if len(mods) > 1:
  38. for mod in mods[1:]:
  39. imp = getattr(imp, mod)
  40. return imp
  41. def find_jobs(jobs_dir):
  42. try:
  43. return [f[:-3] for f in os.listdir(jobs_dir) if not f.startswith('_') and f.endswith(".py")]
  44. except OSError:
  45. return []
  46. def find_job_module(app_name, when=None):
  47. parts = app_name.split('.')
  48. parts.append('jobs')
  49. if when:
  50. parts.append(when)
  51. parts.reverse()
  52. path = None
  53. while parts:
  54. part = parts.pop()
  55. f, path, descr = find_module(part, path and [path] or None)
  56. return path
  57. def import_job(app_name, name, when=None):
  58. jobmodule = "%s.jobs.%s%s" % (app_name, when and "%s." % when or "", name)
  59. job_mod = my_import(jobmodule)
  60. # todo: more friendly message for AttributeError if job_mod does not exist
  61. try:
  62. job = job_mod.Job
  63. except AttributeError:
  64. raise JobError("Job module %s does not contain class instance named 'Job'" % jobmodule)
  65. if when and not (job.when == when or job.when is None):
  66. raise JobError("Job %s is not a %s job." % (jobmodule, when))
  67. return job
  68. def get_jobs(when=None, only_scheduled=False):
  69. """
  70. Return a dictionary mapping of job names together with their respective
  71. application class.
  72. """
  73. # FIXME: HACK: make sure the project dir is on the path when executed as ./manage.py
  74. try:
  75. cpath = os.path.dirname(os.path.realpath(sys.argv[0]))
  76. ppath = os.path.dirname(cpath)
  77. if ppath not in sys.path:
  78. sys.path.append(ppath)
  79. except Exception:
  80. pass
  81. _jobs = {}
  82. for app_name in [app.name for app in apps.get_app_configs()]:
  83. scandirs = (None, 'minutely', 'quarter_hourly', 'hourly', 'daily', 'weekly', 'monthly', 'yearly')
  84. if when:
  85. scandirs = None, when
  86. for subdir in scandirs:
  87. try:
  88. path = find_job_module(app_name, subdir)
  89. for name in find_jobs(path):
  90. if (app_name, name) in _jobs:
  91. raise JobError("Duplicate job %s" % name)
  92. job = import_job(app_name, name, subdir)
  93. if only_scheduled and job.when is None:
  94. # only include jobs which are scheduled
  95. continue
  96. if when and job.when != when:
  97. # generic job not in same schedule
  98. continue
  99. _jobs[(app_name, name)] = job
  100. except ImportError:
  101. # No job module -- continue scanning
  102. pass
  103. return _jobs
  104. def get_job(app_name, job_name):
  105. jobs = get_jobs()
  106. if app_name:
  107. return jobs[(app_name, job_name)]
  108. else:
  109. for a, j in jobs.keys():
  110. if j == job_name:
  111. return jobs[(a, j)]
  112. raise KeyError("Job not found: %s" % job_name)
  113. def print_jobs(when=None, only_scheduled=False, show_when=True, show_appname=False, show_header=True):
  114. jobmap = get_jobs(when, only_scheduled=only_scheduled)
  115. print("Job List: %i jobs" % len(jobmap))
  116. jlist = sorted(jobmap.keys())
  117. if not jlist:
  118. return
  119. appname_spacer = "%%-%is" % max(len(e[0]) for e in jlist)
  120. name_spacer = "%%-%is" % max(len(e[1]) for e in jlist)
  121. when_spacer = "%%-%is" % max(len(e.when) for e in jobmap.values() if e.when)
  122. if show_header:
  123. line = " "
  124. if show_appname:
  125. line += appname_spacer % "appname" + " - "
  126. line += name_spacer % "jobname"
  127. if show_when:
  128. line += " - " + when_spacer % "when"
  129. line += " - help"
  130. print(line)
  131. print("-" * 80)
  132. for app_name, job_name in jlist:
  133. job = jobmap[(app_name, job_name)]
  134. line = " "
  135. if show_appname:
  136. line += appname_spacer % app_name + " - "
  137. line += name_spacer % job_name
  138. if show_when:
  139. line += " - " + when_spacer % (job.when and job.when or "")
  140. line += " - " + job.help
  141. print(line)