I have three loggers logging to the same file
Case1 here i have defined handler only once and added it to all loggers later. It shows the correct results
import pytz
import datetime
import logging
class Formatter(logging.Formatter):
"""override logging.Formatter to use an aware datetime object"""
def converter(self, timestamp):
#dt = datetime.datetime.fromtimestamp(timestamp)
#we use
dt = datetime.datetime.utcnow()
current_time_utc = pytz.utc.localize(dt)
tzinfo = pytz.timezone('America/New_York')
current_time_time_zone = current_time_utc.astimezone(tzinfo)
#print(current_time_time_zone)
return current_time_time_zone
def formatTime(self, record, datefmt=None):
dt = self.converter(record.created)
if datefmt:
s = dt.strftime(datefmt)
else:
try:
s = dt.isoformat(timespec='milliseconds')
except TypeError:
s = dt.isoformat()
return s
import logging
from logging.handlers import RotatingFileHandler
filename1 = "./file.log"
handler = RotatingFileHandler(filename1, mode='a', maxBytes=400, backupCount=10, encoding='utf-8', delay=0)
handler.setLevel(logging.INFO)
handler.setFormatter(Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
# logger1
logger1 = logging.getLogger('name1')
logger1.setLevel(logging.INFO)
logger1.addHandler(handler)
#logger2
logger2 = logging.getLogger('name2')
logger2.setLevel(logging.INFO)
logger2.addHandler(handler)
#logger3
logger3 = logging.getLogger('name3')
logger3.setLevel(logging.INFO)
logger3.addHandler(handler)
for i in range(0,100):
logger1.info("form logging1")
logger2.info("form logging2")
logger3.info("form logging3")
# run this command to concatenate the logs
# python test.py; find ./ -name "file.*" | sort -r -V | xargs awk '(FNR==1){print ">> " FILENAME " <<"}1' > all_logs_test.txt
results: all logs contenated
>> ./file.log.10 <<
2021-03-09T22:48:42.229-05:00:::INFO :::form logging1
2021-03-09T22:48:42.229-05:00:::INFO :::form logging2
2021-03-09T22:48:42.229-05:00:::INFO :::form logging3
2021-03-09T22:48:42.229-05:00:::INFO :::form logging1
2021-03-09T22:48:42.230-05:00:::INFO :::form logging2
2021-03-09T22:48:42.230-05:00:::INFO :::form logging3
>> ./file.log.9 <<
2021-03-09T22:48:42.230-05:00:::INFO :::form logging1
2021-03-09T22:48:42.230-05:00:::INFO :::form logging2
2021-03-09T22:48:42.230-05:00:::INFO :::form logging3
2021-03-09T22:48:42.230-05:00:::INFO :::form logging1
2021-03-09T22:48:42.230-05:00:::INFO :::form logging2
2021-03-09T22:48:42.231-05:00:::INFO :::form logging3
>> ./file.log.8 <<
2021-03-09T22:48:42.231-05:00:::INFO :::form logging1
2021-03-09T22:48:42.231-05:00:::INFO :::form logging2
2021-03-09T22:48:42.231-05:00:::INFO :::form logging3
2021-03-09T22:48:42.231-05:00:::INFO :::form logging1
2021-03-09T22:48:42.231-05:00:::INFO :::form logging2
2021-03-09T22:48:42.231-05:00:::INFO :::form logging3
>> ./file.log.7 <<
2021-03-09T22:48:42.232-05:00:::INFO :::form logging1
2021-03-09T22:48:42.232-05:00:::INFO :::form logging2
2021-03-09T22:48:42.232-05:00:::INFO :::form logging3
2021-03-09T22:48:42.232-05:00:::INFO :::form logging1
2021-03-09T22:48:42.233-05:00:::INFO :::form logging2
2021-03-09T22:48:42.233-05:00:::INFO :::form logging3
>> ./file.log.6 <<
2021-03-09T22:48:42.233-05:00:::INFO :::form logging1
2021-03-09T22:48:42.233-05:00:::INFO :::form logging2
2021-03-09T22:48:42.233-05:00:::INFO :::form logging3
2021-03-09T22:48:42.234-05:00:::INFO :::form logging1
2021-03-09T22:48:42.234-05:00:::INFO :::form logging2
2021-03-09T22:48:42.234-05:00:::INFO :::form logging3
>> ./file.log.5 <<
2021-03-09T22:48:42.234-05:00:::INFO :::form logging1
2021-03-09T22:48:42.234-05:00:::INFO :::form logging2
2021-03-09T22:48:42.235-05:00:::INFO :::form logging3
2021-03-09T22:48:42.235-05:00:::INFO :::form logging1
2021-03-09T22:48:42.235-05:00:::INFO :::form logging2
2021-03-09T22:48:42.235-05:00:::INFO :::form logging3
>> ./file.log.4 <<
2021-03-09T22:48:42.235-05:00:::INFO :::form logging1
2021-03-09T22:48:42.236-05:00:::INFO :::form logging2
2021-03-09T22:48:42.236-05:00:::INFO :::form logging3
2021-03-09T22:48:42.236-05:00:::INFO :::form logging1
2021-03-09T22:48:42.236-05:00:::INFO :::form logging2
2021-03-09T22:48:42.236-05:00:::INFO :::form logging3
>> ./file.log.3 <<
2021-03-09T22:48:42.237-05:00:::INFO :::form logging1
2021-03-09T22:48:42.237-05:00:::INFO :::form logging2
2021-03-09T22:48:42.237-05:00:::INFO :::form logging3
2021-03-09T22:48:42.237-05:00:::INFO :::form logging1
2021-03-09T22:48:42.237-05:00:::INFO :::form logging2
2021-03-09T22:48:42.237-05:00:::INFO :::form logging3
>> ./file.log.2 <<
2021-03-09T22:48:42.238-05:00:::INFO :::form logging1
2021-03-09T22:48:42.238-05:00:::INFO :::form logging2
2021-03-09T22:48:42.238-05:00:::INFO :::form logging3
2021-03-09T22:48:42.238-05:00:::INFO :::form logging1
2021-03-09T22:48:42.238-05:00:::INFO :::form logging2
2021-03-09T22:48:42.238-05:00:::INFO :::form logging3
>> ./file.log.1 <<
2021-03-09T22:48:42.239-05:00:::INFO :::form logging1
2021-03-09T22:48:42.239-05:00:::INFO :::form logging2
2021-03-09T22:48:42.239-05:00:::INFO :::form logging3
2021-03-09T22:48:42.239-05:00:::INFO :::form logging1
2021-03-09T22:48:42.239-05:00:::INFO :::form logging2
2021-03-09T22:48:42.239-05:00:::INFO :::form logging3
>> ./file.log <<
2021-03-09T22:48:42.240-05:00:::INFO :::form logging1
2021-03-09T22:48:42.240-05:00:::INFO :::form logging2
2021-03-09T22:48:42.240-05:00:::INFO :::form logging3
case2:
Whereas here i define handler in each and then create loggers, not showing results correctly
import pytz
import datetime
import logging
class Formatter(logging.Formatter):
"""override logging.Formatter to use an aware datetime object"""
def converter(self, timestamp):
#dt = datetime.datetime.fromtimestamp(timestamp)
#we use
dt = datetime.datetime.utcnow()
current_time_utc = pytz.utc.localize(dt)
tzinfo = pytz.timezone('America/New_York')
current_time_time_zone = current_time_utc.astimezone(tzinfo)
#print(current_time_time_zone)
return current_time_time_zone
def formatTime(self, record, datefmt=None):
dt = self.converter(record.created)
if datefmt:
s = dt.strftime(datefmt)
else:
try:
s = dt.isoformat(timespec='milliseconds')
except TypeError:
s = dt.isoformat()
return s
import logging
from logging.handlers import RotatingFileHandler
filename1 = "./file.log"
# logger1
logger1 = logging.getLogger('name1')
logger1.setLevel(logging.INFO)
handler = RotatingFileHandler(filename1, mode='a', maxBytes=400, backupCount=10, encoding='utf-8', delay=0)
handler.setLevel(logging.INFO)
handler.setFormatter(Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger1.addHandler(handler)
#logger2
logger2 = logging.getLogger('name2')
logger2.setLevel(logging.INFO)
handler = RotatingFileHandler(filename1, mode='a', maxBytes=400, backupCount=10, encoding='utf-8', delay=0)
handler.setLevel(logging.INFO)
handler.setFormatter(Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger2.addHandler(handler)
#logger3
logger3 = logging.getLogger('name3')
logger3.setLevel(logging.INFO)
handler = RotatingFileHandler(filename1, mode='a', maxBytes=400, backupCount=10, encoding='utf-8', delay=0)
handler.setLevel(logging.INFO)
handler.setFormatter(Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger3.addHandler(handler)
for i in range(0,100):
logger1.info("form logging1")
logger2.info("form logging2")
logger3.info("form logging3")
# run this command to concatenate
# python test_2.py; find ./ -name "file.*" | sort -r -V | xargs awk '(FNR==1){print ">> " FILENAME " <<"}1' > all_logs_test_2.txt
results: (when run in linux)
>> ./file.log.10 <<
2021-03-09T22:42:43.864-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.866-05:00::LOGGER1:::INFO :::form logging3
>> ./file.log.9 <<
2021-03-09T22:42:43.864-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.865-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.866-05:00::LOGGER1:::INFO :::form logging1
>> ./file.log.8 <<
2021-03-09T22:42:43.866-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging2
>> ./file.log.7 <<
2021-03-09T22:42:43.867-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging3
>> ./file.log.6 <<
2021-03-09T22:42:43.867-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.868-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.869-05:00::LOGGER1:::INFO :::form logging1
>> ./file.log.5 <<
2021-03-09T22:42:43.869-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging2
>> ./file.log.4 <<
2021-03-09T22:42:43.869-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging3
>> ./file.log.3 <<
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.870-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging1
>> ./file.log.2 <<
2021-03-09T22:42:43.871-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.872-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging2
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging2
>> ./file.log.1 <<
2021-03-09T22:42:43.872-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.872-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging3
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging3
>> ./file.log <<
2021-03-09T22:42:43.872-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging1
2021-03-09T22:42:43.873-05:00::LOGGER1:::INFO :::form logging1
Also when i try the second one in windows it gives the following error. It even does not give the result like above
the test2.py gives this error in windows
--- Logging error ---
Traceback (most recent call last):
File "C:\\Users\\dev\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\logging\\handlers.py", line 70, in emit
self.doRollover()
File "C:\\Users\\dev\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\logging\\handlers.py", line 171, in doRollover
self.rotate(self.baseFilename, dfn)
File "C:\\Users\\dev\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\logging\\handlers.py", line 111, in rotate
os.rename(source, dest)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'D:\\\\Dev\\\\Stock-chart\\\\backend_django\\\\stock\\\\cron_jobs\\\\testing2\\\\file.log' -> 'D:\\\\Dev\\\\Stock-chart\\\\backend_django\\\\stock\\\\cron_jobs\\\\testing2\\\\file.log.1'
Call stack:
File "testing2.py", line 65, in <module>
logger2.info("form logging2")
Message: 'form logging2'
Arguments: ()
I think the results should be shown correctly in the second case also, why its behaving like that.
Where Case1 is not practical way to define three loggers seperately
Whereas Case2 is more practical to define each logger in a seperate function
So coding point of view Case2 is more practical. Like i can have three functions, pass the filename and they will create three loggers
What is working for me to have multiple loggers log to single file without hicups and also using rotationfilehandler.
I wanted to use to for module level. BUt i found that it keeps duplicating the logs. So this helped me to avoid that
for hdlr in logger.handlers[:]: # remove all old handlers
logger.removeHandler(hdlr)
the final working code
so i have logging_config.py
import pytz
import datetime
import logging
from logging.handlers import RotatingFileHandler
def create_handler(filename):
#filename1 = "./file.log"
handler = RotatingFileHandler(filename, mode='a', maxBytes=1000, backupCount=10, encoding='utf-8', delay=0)
#handler = logging.FileHandler(filename,mode='a')
return handler
# we have different logger to have differnt formats. here it may look all are same and of no use
# to create three. Like we can have sepearte logger for sql and requests and normal logging
def get_logger1(handler):
logger1 = logging.getLogger('name1')
logger1.setLevel(logging.INFO)
for hdlr in logger1.handlers[:]: # remove all old handlers
logger1.removeHandler(hdlr)
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger1.addHandler(handler)
return logger1
def get_logger2(handler):
logger2 = logging.getLogger('name2')
logger2.setLevel(logging.INFO)
for hdlr in logger2.handlers[:]: # remove all old handlers
logger2.removeHandler(hdlr)
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger2.addHandler(handler)
return logger2
def get_logger3(handler):
logger3 = logging.getLogger('name3')
logger3.setLevel(logging.INFO)
for hdlr in logger3.handlers[:]: # remove all old handlers
logger3.removeHandler(hdlr)
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter("\n%(asctime)s:::%(levelname)-8s:::%(message)s"))
logger3.addHandler(handler)
return logger3
test1.py
import time
from test2 import logger_test
from logging_config import get_logger1, get_logger2, get_logger3, create_handler
import logging
handler = create_handler("file.log")
logger1 = get_logger1(handler)
logger2 = get_logger2(handler)
logger3 = get_logger3(handler)
for i in range(0,10):
logger1.info("form logging1")
time.sleep(1)
logger2.info("form logging2")
time.sleep(1)
logger3.info("form logging3")
time.sleep(1)
logger_test()
test2.py
import time
from logging_config import get_logger1, get_logger2, get_logger3, create_handler
import logging
from logging.handlers import RotatingFileHandler
handler = create_handler("file.log")
logger1 = get_logger1(handler)
logger2 = get_logger2(handler)
logger3 = get_logger3(handler)
def logger_test():
logger1.info("form logging1 another file")
time.sleep(1)
logger2.info("form logging2 another file")
time.sleep(1)
logger3.info("form logging3 another file")
time.sleep(1)
and eveything is printed in sequence in multiple files
>> ./file.log.3 <<
2021-03-10T02:10:56.657-05:00:::INFO :::form logging1
2021-03-10T02:10:57.659-05:00:::INFO :::form logging2
2021-03-10T02:10:58.661-05:00:::INFO :::form logging3
2021-03-10T02:10:59.663-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:00.666-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:01.667-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:02.669-05:00:::INFO :::form logging1
2021-03-10T02:11:03.671-05:00:::INFO :::form logging2
2021-03-10T02:11:04.673-05:00:::INFO :::form logging3
2021-03-10T02:11:05.674-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:06.676-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:07.678-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:08.680-05:00:::INFO :::form logging1
2021-03-10T02:11:09.681-05:00:::INFO :::form logging2
2021-03-10T02:11:10.683-05:00:::INFO :::form logging3
>> ./file.log.2 <<
2021-03-10T02:11:11.686-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:12.687-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:13.688-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:14.690-05:00:::INFO :::form logging1
2021-03-10T02:11:15.692-05:00:::INFO :::form logging2
2021-03-10T02:11:16.695-05:00:::INFO :::form logging3
2021-03-10T02:11:17.697-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:18.699-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:19.701-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:20.703-05:00:::INFO :::form logging1
2021-03-10T02:11:21.704-05:00:::INFO :::form logging2
2021-03-10T02:11:22.706-05:00:::INFO :::form logging3
2021-03-10T02:11:23.707-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:24.709-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:25.711-05:00:::INFO :::form logging3 another file
>> ./file.log.1 <<
2021-03-10T02:11:26.713-05:00:::INFO :::form logging1
2021-03-10T02:11:27.715-05:00:::INFO :::form logging2
2021-03-10T02:11:28.717-05:00:::INFO :::form logging3
2021-03-10T02:11:29.719-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:30.721-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:31.723-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:32.724-05:00:::INFO :::form logging1
2021-03-10T02:11:33.726-05:00:::INFO :::form logging2
2021-03-10T02:11:34.728-05:00:::INFO :::form logging3
2021-03-10T02:11:35.729-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:36.730-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:37.732-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:38.734-05:00:::INFO :::form logging1
2021-03-10T02:11:39.735-05:00:::INFO :::form logging2
2021-03-10T02:11:40.737-05:00:::INFO :::form logging3
>> ./file.log <<
2021-03-10T02:11:41.740-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:42.741-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:43.743-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:44.744-05:00:::INFO :::form logging1
2021-03-10T02:11:45.746-05:00:::INFO :::form logging2
2021-03-10T02:11:46.748-05:00:::INFO :::form logging3
2021-03-10T02:11:47.749-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:48.751-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:49.752-05:00:::INFO :::form logging3 another file
2021-03-10T02:11:50.754-05:00:::INFO :::form logging1
2021-03-10T02:11:51.756-05:00:::INFO :::form logging2
2021-03-10T02:11:52.758-05:00:::INFO :::form logging3
2021-03-10T02:11:53.760-05:00:::INFO :::form logging1 another file
2021-03-10T02:11:54.761-05:00:::INFO :::form logging2 another file
2021-03-10T02:11:55.763-05:00:::INFO :::form logging3 another file
Related
I am trying to download some Avro files from S3 using Flink. But I am getting the below error not sure why. I have add the aws-java-sdk-core-1.11.342.jar and hadoop-aws-2.7.4.jar jar file to the flink lib directory. The program runs fine when there is no s3 interaction, but gives the error messages when trying to interact with s3. What could be the issue here ?
Flink cluster version : 1.3.2
SBT dependency:-
scalaVersion in ThisBuild := "2.11.11"
val flink_version = "1.3.2"
val slf4j_version = "1.7.25"
val playJson_version = "2.6.3"
val avro_version = "1.8.2"
val cassandra_version = "3.3.0"
val mongodb_version = "3.5.0"
val commons_version = "1.40"
val aws_sdk_version = "1.11.342"
val dependencies = Seq(
"org.apache.flink" %% "flink-scala" % flink_version,
"org.apache.flink" %% "flink-clients" % flink_version,
"org.apache.flink" %% "flink-avro" % flink_version,
"org.apache.hadoop" % "hadoop-aws" % "2.7.4" % Provided,
"org.apache.flink" %% "flink-hadoop-compatibility" % flink_version,
"org.apache.flink" %% "flink-connector-cassandra" % flink_version,
"org.mongodb" % "mongo-java-driver" % mongodb_version,
"com.typesafe.play" %% "play-json" % playJson_version,
"org.apache.avro" % "avro-mapred" % "1.7.1",
"com.paulgoldbaum" %% "scala-influxdb-client" % "0.4.3",
"com.amazonaws" % "aws-java-sdk-dynamodb" % aws_sdk_version,
"com.amazonaws" % "aws-java-sdk-core" % aws_sdk_version % Provided
) ++ Seq(
"org.specs2" %% "specs2-core" % "3.9.4" % Test,
"org.mockito" % "mockito-all" % "1.10.19" % Test,
"com.amazonaws" % "DynamoDBLocal" % "[1.11,2.0)" % Test,
"org.slf4j" % "slf4j-api" % slf4j_version,
"org.slf4j" % "slf4j-log4j12" % slf4j_version
)
Error:-
at org.apache.flink.runtime.jobmanager.JobManager.aroundReceive(JobManager.scala:125)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: org.apache.flink.runtime.JobException: Creating the input splits caused an error: com/amazonaws/services/s3/AmazonS3
at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:247)
at org.apache.flink.runtime.executiongraph.ExecutionGraph.attachJobGraph(ExecutionGraph.java:780)
at org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder.buildGraph(ExecutionGraphBuilder.java:166)
at org.apache.flink.runtime.jobmanager.JobManager.org$apache$flink$runtime$jobmanager$JobManager$$submitJob(JobManager.scala:1315)
... 19 more
Caused by: java.lang.NoClassDefFoundError: com/amazonaws/services/s3/AmazonS3
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:1821)
at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1786)
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1880)
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2298)
at org.apache.flink.runtime.fs.hdfs.HadoopFileSystem.getHadoopWrapperClassNameForFileSystem(HadoopFileSystem.java:486)
at org.apache.flink.core.fs.FileSystem.getHadoopWrapperClassNameForFileSystem(FileSystem.java:437)
at org.apache.flink.core.fs.FileSystem.getUnguardedFileSystem(FileSystem.java:339)
at org.apache.flink.core.fs.FileSystem.get(FileSystem.java:389)
at org.apache.flink.core.fs.Path.getFileSystem(Path.java:293)
at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:472)
at org.apache.flink.api.common.io.FileInputFormat.createInputSplits(FileInputFormat.java:62)
at org.apache.flink.runtime.executiongraph.ExecutionJobVertex.<init>(ExecutionJobVertex.java:233)
... 22 more
Caused by: java.lang.ClassNotFoundException: com.amazonaws.services.s3.AmazonS3
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 36 more
I'm trying to remove certain objects from a PDF file. The objects all look like this:
40 0 obj
<<
/PieceInfo
/Subtype /Form
/Resources
<<
/Font
<<
/Fm1 35 0 R
>>
>>
/Type /XObject
/BBox [0 -22.5 131.05 0]
/Length 601
/Matrix [1 0 0 1 0 0]
>>
stream
. . .
A bunch of compressed gibberish here
. . .
endstream
endobj
What I found to work without breaking the PDF document is deleting the stuff between obj, stream, and endstream.
Is there a way in sed or awk to look for lines containing /Form, and then deleting everything between the nearest obj above and stream below, and that stream and the endstream below it, so that the final result looks like this:
40 0 obj
stream
endstream
endobj
Given:
$ echo "$pdf"
40 0 obj
<<
/PieceInfo
/Subtype /Form
/Resources
<<
/Font
<<
/Fm1 35 0 R
>>
>>
/Type /XObject
/BBox [0 -22.5 131.05 0]
/Length 601
/Matrix [1 0 0 1 0 0]
>>
stream
. . .
A bunch of compressed gibberish here
. . .
endstream
endobj
You can use perl:
$ echo "$pdf" | perl -0777 -lne 'print "$1$2$3\n" if /(^.*(?<=\bobj)\s*\R)[\s\S]*?\/Form[\s\S]*?^(stream\s*^)[\s\S]*?^(endstream\s+endobj)/m'
40 0 obj
stream
endstream
endobj
Demo and explanation of regex
perl -0777 -pe 's/(?<=obj)[\s\S]+?\/Form[\s\S]+?\n(?=endstream)/\nstream\n/g' pdf
There are a LOT of ways this regex can backfire (The key issue is "obj" or "endstream" appearing midstream or those fields or "/Form" being missing). You'll need a full script for something production quality, in which case you would definitely need to "show your work" to get help. Also you may need to remove or change the \n before (?=endstream) for an actual PDF. I'm not familiar with the line end characters it uses.
The jist, as a glob, would be that it looks for obj*/Form*endstream, and then clobbers everything not in a lookaround (?[etc]), and it manually readds the stream line.
awk may also do the job,
awk '/[^end]obj/||/[end]*stream/{print;if(d==1){s=""}d=1;next}{s=s $0}END{print s}' pdf
Brief explanation,
/[^end]obj/||/[end]*stream/: locate the string 'obj', 'stream', and 'endstream'
If above string is existed in the line, print it and enable the flag d
If the d is already enabled, clear the buffer str
Print the str in the end
This might work for you (GNU sed):
sed -r '/\<obj\>/{n;:a;/\<endobj\>/!{N;ba};s/.*\<(stream)\>.*\<(endobj)\>/\1\n\2/}' file
Gather up the lines between obj and endobj and remove the parts either side of stream.
$ cat tst.awk
$NF == "endobj" {
print (obj ~ "/Form" ? "stream" ORS "endstream" : obj)
obj = ""
inObj = 0
}
inObj { obj = (obj == "" ? "" : obj ORS) $0 }
!inObj { print }
$NF == "obj" { inObj = 1 }
$ awk -f tst.awk file
40 0 obj
stream
endstream
endobj
I'm trying to write a code which adds hyperlinks to existing PDF document.
I've tried to do that by following PDF 32000 specification but seems that I'm missing something.
PDF viewer shows my link annotations and changes the mouse pointer to "finger" when hovering over them, but no URLs are shown, and clicking on them doesn't do anything.
PDF contents are shown below.
Page:
/Type/Page
/Parent 1 0 R
/MediaBox[0 0 594.75 841.5]
/Contents 7 0 R
/Resources 8 0 R
/Annots [5 0 R 6 0 R] % added by me
First annotation (for "Google", #5):
/Type /Annot
/Subtype /Link
/Rect [60.0 779.25 150.0 767.25]
/BS
<<
/W 2 % should be 0, set to 2 to be able to see the annotation's position
>>
/F 4
/A
<<
/Type /Action
/Subtype /URI
/URI (https://google.com)
>>
Second annotation (for "DuckDuckGo", #6):
/Type /Annot
/Subtype /Link
/Rect [342.0 694.5 432.0 682.5]
/BS
<<
/W 2 % should be 0, set to 2 to be able to see the annotation's position
>>
/F 4
/A
<<
/Type /Action
/Subtype /URI
/URI (https://duckduckgo.com)
>>
Content stream:
1 0 0 -1 36.0 841.5 cm
-100 Tz
q
/Font3 -9.0 Tf
0.0 0.0 0.0 rg
24.0 62.25 90.0 12.0 re W n
BT
1 0 0 1 0 0 Tm
24.75 69.75 Td
(Google) Tj
ET
Q
q
/Font3 -9.0 Tf
0.0 0.0 0.0 rg
306.0 147.0 90.0 12.0 re W n
BT
1 0 0 1 0 0 Tm
306.75 154.5 Td
(DuckDuckGo) Tj
ET
Q
This code produces the following result:
Generated PDF file is available here.
I took a look on the code the Microsoft Word produces for the links, and it looks pretty much the same (except minification and structural parent; this code works, but mine is not):
/Subtype/Link/Rect[ 69.75 697.51 113.16 720] /BS
<<
/W 0
>>
/F 4/A
<<
/Type/Action/S/URI/URI(https://google.com/)
>>
/StructParent 0
Any advice will be much appreciated.
Your action dictionary looks like this:
<<
/Type /Action
/Subtype /URI
/URI (https://google.com)
>>
If you lookup ISO 32000-1 section 12.6.2 Action Dictionaries, though, you'll see that the type of action is not the value of Subtype but instead of S:
S
name
(Required) The type of action that this dictionary describes; see Table 194 for specific values.
(ISO 32000-1 Table 193 – Entries common to all action dictionaries)
Thus, in your Action dictionary (not in the surrounding Annot dictionary, though!) replace /Subtype by /S:
/Type /Annot
/Subtype /Link %<------------- not here
/Rect [342.0 694.5 432.0 682.5]
/BS
<<
/W 2 % should be 0, set to 2 to be able to see the annotation's position
>>
/F 4
/A
<<
/Type /Action
/S /URI %<------------- but here
/URI (https://duckduckgo.com)
>>
You have to parse the page content and process all operators that affect text position in order to compute the exact position of Google and DuckDuckGo. Then place the link annotations at the computed positions.
I am looking for a way to draw a dot (or small circle) that is an internal "annotation" link to a popup box in a PDF using specification 1.7...
TCPDF does this internally, with code like... see: https://tcpdf.org/examples/example_036/
2 0 obj
<< /ProcSet [/PDF /Text /ImageB /ImageC /ImageI] /Font << /F1 3 0 R /F2 4 0 R /F3 6 0 R >> /XObject << /XT5 5 0 R /I0 11 0 R >> >>
endobj
7 0 obj
<</Type /Annot /Subtype /Text /Rect [235.275591 737.008110 263.622047 765.354567] /Contents (þÿ
I am looking for a solution that does this directly. That is, explaining how the PDF language creates the trigger object (e.g., the circle), and the annotation object (e.g., the popup box), and how the internal linking works.
I want to do is to create the pdf report using jsreport with web api. All I have got is the response from jsreport server as the pdf content in the following format:
%PDF-1.4 1 0 obj << /Title (þÿ) /Creator (þÿ) /Producer (þÿQt 4.8.4 \(C\) 2011 Nokia Corporation and/or its subsidiary\(-ies\)) /CreationDate (D:20151118135420+05'45') >> endobj 3 0 obj << /Type /ExtGState /SA true /SM 0.02 /ca 1.0 /CA 1.0 /AIS false /SMask /None>> endobj 4 0 obj [/Pattern /DeviceRGB] endobj 5 0 obj << /Type /Page /Parent 2 0 R /Contents 8 0 R /Resources 10 0 R /Annots 11 0 R /MediaBox [0 0 595 842] >> endobj 10 0 obj << /ColorSpace << /PCSp 4 0 R /CSp /DeviceRGB /CSpg /DeviceGray >> /ExtGState << /GSa 3 0 R >> /Pattern << >> /Font << /F6 6 0 R /F7 7 0 R >> /XObject << >> >> endobj 11 0 obj [ ] endobj 8 0 obj << /Length 9 0 R /Filter /FlateDecode >> stream xÕOYrÄïý)úl`4,²Ù$ÃÀF2àÁðÁðÁõÚXx÷à¯ïÈî.Uvñ,;ÀÎ+_½?ù2###üÇßþýþ?ÿvÿãûßþçþ÷Ó¿ßÿv7ܯþùa¸_ïïwûíýïÜ}¾ÿ|÷ëݯOÿýåßïVïvÛÕñ?_ÿò·þa=|ùÒósåo¿ÿõîÇã/Þ?ùíý??ý¯ÿ»_ßÿÓÓþrÿ¯ÿöô¯?~çø?î¶ý»Ãá°ßýà¿Ç¬Þa·Ú®OÿÇ*üùøÿºû¿»ÿë×A?®VÍÃÃf{ôøÏß;èÏ/_¾þ3þjò·¿üûÃîþé³ÿý»?a¯_ýùÌþt÷ãÇÇûÍãý§?ß¿ì/ÿúôÇÝÕ8ÜúÓýß?-ùð÷þr·þ2Ó'ë¯ ¯l¾~r8óÁÃñ;¯l¿~²ýàqúÁîøËÃë'ûãSFç0ý¡e}æ¹Êw~>þ³cóùÒû0U¿$¾õücx͸*Ǧ"þR|ñøSáÁÃöÑpÚÁÚ3S<·q>|z:Ü=góq½Îa=¡ÙO¾éÙÌ÷xzþn½{wü£á&=ì eôù7÷Êîaÿnûõ&ùzLÿ8þVòr1½Î÷Ü1ßA¹jôpß8ÃñòX¯ÎÙ©øIÂVhÖ-Ìë6l§V(~ºòv©xÅÏßq±ó?ã0-25$O/ò¸Únvô³åý±`?¾g¨ß{׳ÇqøÖ Fàiï.Ô£KµîÀuÜßõg6ñh áöO ¿ó~zZãw׿ó|2G¿´?ñÓáâe-ýtxÄ/Ñ »ðW>ÌÃá;ûðNa®ÂªÄÕ_Kg/l<Þ!á+qüq%o;¦£ ?ésÏâÍß1LÌKXÿ÷a8ñ¥âÄ©»f:oøãütxÞ°iOãùáñáùç>]û~6<¿²îãv;ãOaöyûd5hÿ8Q ×ÓYÛO83æ v.ýÃêì¼Æ1>LmDxÞ¾Ï ¯µ[§ÿ½Ú<ÿÜ05Zëéµñ&âh-}g7ýà¸à#ñð¶9-=#\!ÇӹŠNð¹ ãøHã8ù¿#Ó6L¿~6ÌizÇté¯Ä>#þðаcÂ8Âò:sgÞîOø°´Ra×I =üJxhøÙ°Ó}ùì¸lôþøå¤Â)~wUj82áð0Í<ôéF ç0qÊ·åSÖ²þºqù§s8v{¼ã¬ðÚ3óÓtìü³áí¦Ï`Û0Ìa§²nª8ôðaµÑü 7Wõ`Aø]²V¨Ù Zof7 n¥8SÓg:¼v8äaø8"½Õ§%-|c½qíf0ЧP%àç<¸º_ôÆË -¦£~'C¸Ó¾ sX¿f0¬CÝrƾeâ#=N{8aË S¼l3nݺ?wHXÌ°1îGr·-z,GQ¹ùõ9G÷#~%ÌXp8è ËÀ±wX0|²KÃ{½BËýÆNð%^GpÙp(¡C°CÅK3sË6{hÙÝì/Ca¦Âé <³#_ûq5~ïõöîµ éH}Kù¸ÃÏ^ÉH0I}ÞS a°§Î`æw +<w<0=#°(o*üÄyoÇþ°!]oÂërÀQ_aÓu(ñ³Sw qû c«#Ø×_ÆVRøú$QKÖìÊÐÉfoðq3ûWkÁùV4Dçñ+xP,y !ELè`gÌçâï²WNßWÕU°ðAø ÁÄIÅ=®.ï¡°v[:Ë×ù%ì/?2ÇN` ðëò3ØáóÀ§.<#¸Zg&ìÁãtî¯Øv!ÁÃç!ø ìr`¸3-ubdxb,2½uã¼!ïr¨LHN|}¾|³Ï·öù TÂMä+]ýàì7>=r çâ½ÀN_=XfO§×+HXæ?µämä 5Ï0a`ÁmâÚ:Èã³,áÔ¯¸ø³* å ñfT(í%lÙx¤LæIõÁÕ·ÄÚOÜdÃÌ.$G&(|Ä8fd7áu Øá µøÎ>ÉqzµõKYA©:þ= 4ÊÄÙ)XrÓÿ®=ØëðÄÐ Ü\à¬ÀLg_]pøêì° ×æò ÍvøÆXçLÀùUî²±HÕcÁDØʶ`h¾.#bJº#n è ÖÆ$üÞÿ<elÕy3;*Ð/ÔFÔ«ú\kÖ/;÷ú¼ØÌ^Îùfÿð¥àâ ·_Â#qÊNÀæ8:g$2 ½G¯^DêHXgJ,õ[J ¾»Ä[®Ûì'T´õåéáðÅ#NdàùípÊùeNã1öÎøÚH%Ñé4^H×8¬&ÿ ;\·â(da÷v½ÒAÆ,b_ Ù5ztv°Â;;%w.S&×¢8!j·ÃhO õÌdT¯T'/Ëà qnQÿt Àm·[ 7B&ØÔ]âõTéMqQG ±¬® $t OaIÔ}Ä55ºZïX¬+g3¦9¹Ç®cRk dh*ËÜ<c3g½Ù }-±àâº2.Öj¡¥2©²ïI¥R½'5Bi¸!{<7!ë['ê ø¬B å<§ \9N¸¿î£¡.nw2'¸¥ú5äëæ *ðòÕÙ1ÍÅ0lAÉáX¼l(ÄoÑá#Âuæ+kØA3 ƺ<váó^gä*.S&à¹KTÍ®÷óÈ"ë²r_¿:«À¼ ¢%ZâKßRà'x0uÃa^)U¶²·'Töð¶s¼ZýoTx,962¼tB®¬Nsã0Ð/" 5o½à0-$ÜÆ·£rZMUËù*IÐ ëØ ×w ÂÝæD¯!Æ°¶É+qB^UV¹^è'D#'Ú1G|92`ç,-DaÆp®#Ç¥P(Æ7Ñ;ÅÁCGôx]Ci¸Ò¢Cgo¾ !Á¯/ì¹Øÿ¿Loadn;ág5k¶mT{BëË!ú¿#v¹.MÍüH£Rû³¬%!¸K°7¯*ÆÁãfÓjàM%Z8ÔqqÁTÁjOláþ24 ¨N)`¦ìX íJºL^|¥ Å¥p{7l6;cB©·EáÑ¥DTÝ77ÃKa¤B&5<#° ³Ñzâ¶qÄ7wëõØ3XNV=àHë$LcòÜ2K°u$v6Lov÷f)ù½ ³ñ;*¸Âw5%Gõ]îר#¢ô IëßD¥ÙD(ǯþKsç8§X:& 3·# HJV%«wtKÔ9*/rð/dD.õ¼¾LåPÝ^üRG¾4sæÌ5;|³ýÍB9¼¦2H¼Må¡ WwWeH9 ݨX`8±RÎüz~=F¤ã&g,Õ5·c²Î4AS)ñv¥Fô,´oÂ÷2*oÍIx{uf Å·¼W[_$ôÊVøb|È=`ÓÈÈ&MïLTa3°vCdÝ®fçòµuFÝHlêh)9jÇUd9[T¸¹ZÄã9*SùÆvܼÚBp¸¸°E¿¡¤#êNííØTd0/Áª ` CiÇÞ ¨V ³E !ø2¼ØBwKõ õ*#\õLµ45VâEf§;BÌú5 $/âuÔºîºD~4[ÿ×ës>¼$¨TPúÀX£½R<=[|?ô,O>ëyKÉXçá|ù¡ ¹ å*hô0ÞBßâ[༠<:V¹ñ¦p ¡ZbáͶ(IXÈøÒC## Heading
Now I just want to create the pdf file with this content (get as report.content) and save on the server (In Report folder of the project).
jsreport server returned you pdf document in string. simply you have to write this string into file. if you are using php you can do it by
file_put_contents()
sample demo to represent what is returned from jsreport server
// Read the input pdf file in string
$stringPdf = file_get_contents("input.pdf");
echo $stringPdf;
// write string file in pdf
file_put_contents("output.pdf",$stringPdf);
?>