I am having a problem with the StatisticsGen component of TFX.
I use TFRecords and as input data I use RaggedTensors (The TFrecord is created with SequenceExample)
After using ExampleGen which splits the file correctly into train and val, StatisticsGen creates a 0kb file and therefore is not working correctly.
Can anyone confirm me that the RaggedTensors are supported by TFX? If so, can you help me generate the statistics?
TF Version: 2.4.1
Eager mode: True
TFX Version: 0.28.0
TFDV version: 0.28.0
TFT version: 0.28.0
TFMA version: 0.28.0
Hub version: 0.9.0
Beam version: 2.28.0
!pip install tfx
import tensorflow as tf
from tfx.components import ImportExampleGen
from tfx.components import SchemaGen
from tfx.components import StatisticsGen
from tfx.utils.dsl_utils import external_input
from tfx.proto import example_gen_pb2
from tfx.orchestration.experimental.interactive.interactive_context import InteractiveContext
import tfx
import os
import apache_beam as beam
import tensorflow_data_validation as tfdv
import tensorflow_transform as tft
import tensorflow_model_analysis as tfma
import tensorflow_hub as hub
import logging
path_to_test = os.path.join("tf_records")
context = InteractiveContext(pipeline_root='pipeline_root')
#examplegen
example_gen = ImportExampleGen(input_base=path_to_test)
context.run(example_gen, enable_cache=True)
print(example_gen.outputs['examples'].get()[0].uri)
train_uri = os.path.join(example_gen.outputs['examples'].get()[0].uri, 'train')
tfrecord_filenames = [os.path.join(train_uri, name)
for name in os.listdir(train_uri)]
#statisticsgen
statistics_gen = StatisticsGen(
examples=example_gen.outputs['examples'])
context.run(statistics_gen)
context.show(statistics_gen.outputs['statistics'])
In the creation of the TFRecords, instead of using SequenceExample, we solved the issue using Example. Now the StatisticsGen is working, showing the statistics of the dataset.
Related
import os, sys
from google.colab import drive
drive.mount('/content/gdrive')
path = '/content/gdrive/MyDrive/JIW/Libraries'
sys.path.append(path)
# !pip install --target=$path transformers datasets wandb # <- This line can be used to load pkg
from transformers import AutoTokenizer, default_data_collator, AutoModelForQuestionAnswering, TrainingArguments, Trainer
from datasets import Dataset, load_dataset
import wandb
The Transformers/Datasets libraries are saved at the path specified. But the two "from transformers import" and "from datasets import" statements take 5ish minutes to execute on Colab. Is this normal or am I messing up somehow?
I'm trying to follow this repo's tutorial on colabhttps://github.com/divamgupta/image-segmentation-keras
but I'm getting this error again and again
cannot import name 'get_config' from 'tensorflow.python.eager.context' (/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/context.py)
I googled it a lot so I found some solutions that solved same problems for them, like upgrade tensorflow version or using from tensorflow import keras instead of import keras
or add this code snippet
!pip install --upgrade tensorflow
!pip install --upgrade tensorflow-gpu
But none of them worked!
Full error codes are as follow
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-10-5c4963dc4fff> in <module>()
1 from tensorflow import keras
2 from tensorflow.keras.preprocessing import image
----> 3 from keras_segmentation.models.unet import vgg_unet
4
5 model = vgg_unet(n_classes=50 , input_height=320, input_width=640 )
3 frames
/usr/local/lib/python3.7/dist-packages/keras_segmentation/models/unet.py in <module>()
----> 1 from keras.models import *
2 from keras.layers import *
3
4 from .config import IMAGE_ORDERING
5 from .model_utils import get_segmentation_model
/usr/local/lib/python3.7/dist-packages/keras/__init__.py in <module>()
23
24 # See b/110718070#comment18 for more details about this import.
---> 25 from keras import models
26
27 from keras.engine.input_layer import Input
/usr/local/lib/python3.7/dist-packages/keras/models.py in <module>()
17
18 import tensorflow.compat.v2 as tf
---> 19 from keras import backend
20 from keras import metrics as metrics_module
21 from keras import optimizer_v1
/usr/local/lib/python3.7/dist-packages/keras/backend.py in <module>()
35 from tensorflow.python.distribute import distribute_coordinator as dc
36 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
---> 37 from tensorflow.python.eager.context import get_config
38 from tensorflow.python.framework import config
39 from keras import backend_config
ImportError: cannot import name 'get_config' from 'tensorflow.python.eager.context' (/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/context.py)
---------------------------------------------------------------------------
NOTE: If your import is failing due to a missing package, you can
manually install dependencies using either !pip or !apt.
To view examples of installing some common dependencies, click the
"Open Examples" button below.
---------------------------------------------------------------------------
Full codes are as follow
!pip install keras-segmentation
!pip install keras==2.4.3
#!pip install tensorflow==2.4.1
!pip install tensorflow==2.5.0
!apt-get install -y libsm6 libxext6 libxrender-dev
!pip install opencv-python
!pip install --upgrade tensorflow
!pip install --upgrade tensorflow-gpu
! wget https://github.com/divamgupta/datasets/releases/download/seg/dataset1.zip && unzip dataset1.zip
from tensorflow import keras
from tensorflow.keras.preprocessing import image
from keras_segmentation.models.unet import vgg_unet
model = vgg_unet(n_classes=50 , input_height=320, input_width=640 )
model.train(
train_images = "dataset1/images_prepped_train/",
train_annotations = "dataset1/annotations_prepped_train/",
checkpoints_path = "/tmp/vgg_unet_1" , epochs=5
)
out = model.predict_segmentation(
inp="dataset1/images_prepped_test/0016E5_07965.png",
out_fname="/tmp/out.png"
)
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.imshow(out)
from IPython.display import Image
Image('/tmp/out.png')
o = model.predict_segmentation(
inp="dataset1/images_prepped_test/0016E5_07965.png",
out_fname="/tmp/out.png" , overlay_img=True, show_legends=True,
class_names = [ "Sky", "Building", "Pole","Road","Pavement","Tree","SignSymbol", "Fence", "Car","Pedestrian", "Bicyclist"]
)
from IPython.display import Image
Image('/tmp/out.png')
I would appreciate any of your help
thanks in advance
From comments
It was just a matter of version
with tensorflow and keras. I looked into traceback tensorflow error
messages and opened it and changed import keras to from tensorflow import keras issue was resolved (Paraphrased from z2ouu).
Scripts in Tensorflow 1.X,but it's still not working in Tensorflow 2.0
2.How can I migrate this scipts to Tensorflow 2.0
import datetime
import json
import os
import pprint
import random
import string
import sys
import tensorflow as tf
import tensorflow_io as tfio
assert 'COLAB_TPU_ADDR' in os.environ, 'ERROR: Not connected to a TPU runtime; please see the first cell in this notebook for instructions!'
TPU_ADDRESS = 'grpc://' + os.environ['COLAB_TPU_ADDR']
print('TPU address is', TPU_ADDRESS)
from google.colab import auth
auth.authenticate_user()
with tf.compat.v1.Session(TPU_ADDRESS) as session:
print('TPU devices:')
pprint.pprint(session.list_devices())
# Upload credentials to TPU.
with open('/content/adc.json', 'r') as f:
auth_info = json.load(f)
tf.contrib.cloud.configure_gcs(session, credentials=auth_info)
# Now credentials are set for all future sessions on this TPU.
I am creating a notebook on Google Colab to run trading algorithms which execute on Interactive Brokers (IB). I want to use the IB API for this.
My current code downloads and installs the API:
# Install IB and related libraries
!wget -cq http://interactivebrokers.github.io/downloads/twsapi_macunix.975.01.zip
!unzip -qq -o twsapi_macunix.975.01.zip
!cd IBJts/source/pythonclient && python setup.py build install
!pip install ib_insync
!pip install ibapi
# Import generic libraries
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import datetime # For datetime objects
import os.path # To manage paths
import sys # To find out the script name (in argv[0])
import pandas as pd
import numpy as np
import matplotlib as plt
import time
from datetime import datetime
import argparse
# Import IB and IB_InSync, and start an IB connection
from ib_insync import *
from ibapi import *
The last line returns an error:
ModuleNotFoundError Traceback (most recent call last)
<ipython-input-4-3cdef93fab15> in <module>()
48 from ib_insync import *
---> 49 from ibapi import *
ModuleNotFoundError: No module named 'ibapi'
As the installation seems to run properly, I do not understand why I cannot import the API for use in the subsequent code.
Thanks already for your help!
Found a workaround by installing the library directly from the .egg file:
'''
ib_path='/usr/local/lib/python3.6/dist-packages/ibapi-9.75.1-py3.6.egg'
sys.path.append(ib_path)
from ibapi import *
'''
Please help me to solve the problem - I use Google's Colaboratory to run tensor_hub example Text classification with TF-Hub, it needs the version later than 1.7.0, but the version I installed is 1.8.0.
# Install the latest Tensorflow version.
!pip install --quiet "tensorflow>=1.7"
# Install TF-Hub.
!pip install tensorflow-hub
import tensorflow as tf
import tensorflow_hub as hub
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import re
import seaborn as sns
RuntimeErrorTraceback (most recent call last) <ipython-input-20-48e1bdaa8642> in <module>()
1 import tensorflow as tf
----> 2 import tensorflow_hub as hub
3 import matplotlib.pyplot as plt
4 import numpy as np
5 import os
/usr/local/lib/python2.7/dist-packages/tensorflow_hub/__init__.py in <module>()
63
64 # Comment/uncomment to skip checking the TensorFlow version.
---> 65 _check_tensorflow_version(tf.VERSION)
66
67 # Used by doc generation script.
/usr/local/lib/python2.7/dist-packages/tensorflow_hub/__init__.py in
_check_tensorflow_version(version)
60 "TensorFlow Hub depends on 'tf-nightly' build after %s or "
61 "'tensorflow~=%s'. Found tf.VERSION = %s" % (
---> 62 _NIGHTLY_VERSION, _MAIN_VERSION, version))
63
64 # Comment/uncomment to skip checking the TensorFlow version.
RuntimeError: TensorFlow Hub depends on 'tf-nightly' build after 20180308 or 'tensorflow~=1.7'. Found tf.VERSION = 1.6.0
Good news: TF 1.7 is now available by default. :)
The underlying problem you were hitting is just that python caches loaded modules in sys.modules -- re-importing a module doesn't pick up the new version until you restart the process, as Korakot noted.
You may need to restart the runtime.
menu > Runtime > Restart runtime...