Related
This question is a follow-up to "How are zlib, gzip and zip related? What do they have in common and how are they different?" The answers are very detailed but they never quite answer my specific question.
Given a valid GZIP file, should I always be able to extract the deflate-bytes inside and use those bytes to construct a valid PKZIP file with the same contents, without decompressing and recompressing that byte stream?
For example, imagine I have a collection of GZIP files. Could I write a program that quickly (by avoiding deflate/inflate) constructs an equivalent PKZIP file of those files by cutting the GZIP headers off the source files and building a PKZIP structure around the byte streams? (Also the same in reverse by taking any valid PKZIP file and quickly convert them into many GZIP files?)
Both file formats appear to use the same "deflate" algorithm, but is it exactly the same deflate algorithm?
Yes. It is exactly the same deflate format.
(The deflate algorithm can be, and in fact often is different, producing different deflate streams. However that is irrelevant to your application. The format is compatible, and any compliant inflator will be able to decompress the gzip deflate data transplanted into a zip file.)
I forgot why I wrote this, but the C code below will convert a gzip file to a single-entry zip file, with some constraints on the gzip file.
/*
gz2zip.c version 1.0, 31 July 2018
Copyright (C) 2018 Mark Adler
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Mark Adler
madler#alumni.caltech.edu
*/
// Convert gzip (.gz) file to a single entry zip file. See the comments before
// gz2zip() for more details and caveats.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
# include <fcntl.h>
# include <io.h>
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
#else
# define SET_BINARY_MODE(file)
#endif
#define local static
// Exit on error.
local void bail(char *why) {
fprintf(stderr, "gz2zip abort: %s\n", why);
exit(1);
}
// Type to track number of bytes written.
typedef struct {
FILE *out;
off_t off;
} tally_t;
// Write len bytes at dat to t.
local void put(tally_t *t, void const *dat, size_t len) {
size_t ret = fwrite(dat, 1, len, t->out);
if (ret != len)
bail("write error");
t->off += len;
}
// Write 16-bit integer n in little-endian order to t.
local void put2(tally_t *t, unsigned n) {
unsigned char dat[2];
dat[0] = n;
dat[1] = n >> 8;
put(t, dat, 2);
}
// Write 32-bit integer n in little-endian order to t.
local void put4(tally_t *t, unsigned long n) {
put2(t, n);
put2(t, n >> 16);
}
// Write n zeros to t.
local void putz(tally_t *t, unsigned n) {
unsigned char const buf[1] = {0};
while (n--)
put(t, buf, 1);
}
// Convert the Unix time unix to DOS time in the four bytes at *dos. If there
// is a conversion error for any reason, store the current time in DOS format
// at *dos. The Unix time in seconds is rounded up to an even number of
// seconds, since the DOS time can only represent even seconds. If the Unix
// time is before 1980, the minimum DOS time of Jan 1, 1980 is used.
local void unix2dos(unsigned char *dos, time_t unix) {
unix += unix & 1;
struct tm *s = localtime(&unix);
if (s == NULL) {
unix = time(NULL); // on error, use current time
unix += unix & 1;
s = localtime(&unix);
if (s == NULL)
bail("internal error"); // shouldn't happen
}
if (s->tm_year < 80) { // no DOS time before 1980
dos[0] = 0; dos[1] = 0; // use midnight,
dos[2] = (1 << 5) + 1; dos[3] = 0; // Jan 1, 1980
}
else {
dos[0] = (s->tm_min << 5) + (s->tm_sec >> 1);
dos[1] = (s->tm_hour << 3) + (s->tm_min >> 3);
dos[2] = ((s->tm_mon + 1) << 5) + s->tm_mday;
dos[3] = ((s->tm_year - 80) << 1) + ((s->tm_mon + 1) >> 3);
}
}
// Chunk size for reading and writing raw deflate data.
#define CHUNK 16384
// Read the gzip file from in and write it as a single-entry zip file to out.
// This assumes that the gzip file has a single member, that it has no junk
// after the gzip trailer, and that it contains less than 4GB of uncompressed
// data. The gzip file is not decompressed or validated, other than checking
// for the proper header format. The modification time from the gzip header is
// used for the zip entry, unless it is not present, in which case the current
// local time is used for the zip entry. The file name from the gzip header is
// used for the zip entry, unless it is not present, in which case "-" is used.
// This does not use the Zip64 format, so the offsets in the resulting zip file
// must be less than 4GB. If name is not NULL, then the zero-terminated string
// at name is used as the file name for the single entry. Whether the file name
// comes from the gzip header or from name, it is truncated to 64K-1 characters
// if necessary.
//
// It is recommended that unzip -t be used on the resulting file to verify its
// integrity. If the gzip files do not obey the constraints above, then the zip
// file will not be valid.
local void gz2zip(FILE *in, FILE *out, char *name) {
// zip file constant headers for local, central, and end record
unsigned char const loc[] = {'P', 'K', 3, 4, 20, 0, 8, 0, 8, 0};
unsigned char const cen[] = {'P', 'K', 1, 2, 20, 0, 20, 0, 8, 0, 8, 0};
unsigned char const end[] = {'P', 'K', 5, 6, 0, 0, 0, 0, 1, 0, 1, 0};
// gzip header
unsigned char head[10];
// zip file modification date, CRC, and sizes -- initialize to zero for the
// local header (the actual CRC and sizes follow the compressed data)
unsigned char desc[16] = {0};
// name from gzip header to use for the zip entry (the maximum size of the
// name is 64K-1 -- if the gzip name is longer, then it is truncated)
unsigned name_len;
char save[65535];
// read and interpret the gzip header, bailing if it is invalid or has an
// unknown compression method or flag bits set
size_t got = fread(head, 1, sizeof(head), in);
if (got < sizeof(head) ||
head[0] != 0x1f || head[1] != 0x8b || head[2] != 8 || (head[3] & 0xe0))
bail("input not gzip");
if (head[3] & 4) { // extra field (ignore)
unsigned extra = getc(in);
int high = getc(in);
if (high == EOF)
bail("premature end of gzip input");
extra += (unsigned)high << 8;
fread(name, 1, extra, in);
}
if (head[3] & 8) { // file name (save)
name_len = 0;
int ch;
while ((ch = getc(in)) != 0 && ch != EOF)
if (name_len < sizeof(name))
save[name_len++] = ch;
}
else { // no file name
name_len = 1;
save[0] = '-';
}
if (head[3] & 16) { // comment (ignore)
int ch;
while ((ch = getc(in)) != 0 && ch != EOF)
;
}
if (head[3] & 2) { // header crc (ignore)
getc(in);
getc(in);
}
// use name from argument if present, otherwise from gzip header
if (name == NULL)
name = save;
else {
name_len = strlen(name);
if (name_len > 65535)
name_len = 65535;
}
// set modification time and date in descriptor from gzip header
time_t mod = head[4] + (head[5] << 8) + ((time_t)(head[6]) << 16) +
((time_t)(head[7]) << 24);
unix2dos(desc, mod ? mod : time(NULL));
// initialize tally of output bytes
tally_t zip = {out, 0};
// write zip local header
off_t locoff = zip.off;
put(&zip, loc, sizeof(loc));
put(&zip, desc, sizeof(desc));
put2(&zip, name_len);
putz(&zip, 2);
put(&zip, name, name_len);
// copy raw deflate stream, saving eight-byte gzip trailer
unsigned char buf[CHUNK + 8];
if (fread(buf, 1, 8, in) != 8)
bail("premature end of gzip input");
off_t comp = 0;
while ((got = fread(buf + 8, 1, CHUNK, in)) != 0) {
put(&zip, buf, got);
comp += got;
memmove(buf, buf + got, 8);
}
// write descriptor based on gzip trailer and compressed count
memcpy(desc + 4, buf, 4);
desc[8] = comp;
desc[9] = comp >> 8;
desc[10] = comp >> 16;
desc[11] = comp >> 24;
memcpy(desc + 12, buf + 4, 4);
put(&zip, desc + 4, sizeof(desc) - 4);
// write zip central directory
off_t cenoff = zip.off;
put(&zip, cen, sizeof(cen));
put(&zip, desc, sizeof(desc));
put2(&zip, name_len);
putz(&zip, 12);
put4(&zip, locoff);
put(&zip, name, name_len);
// write zip end-of-central-directory record
off_t endoff = zip.off;
put(&zip, end, sizeof(end));
put4(&zip, endoff - cenoff);
put4(&zip, cenoff);
putz(&zip, 2);
}
// Convert the gzip file on stdin to a zip file on stdout. If present, the
// first argument is used as the file name in the zip entry.
int main(int argc, char **argv) {
// avoid end-of-line conversions on evil operating systems
SET_BINARY_MODE(stdin);
SET_BINARY_MODE(stdout);
// convert .gz on stdin to .zip on stdout -- error returns use exit()
gz2zip(stdin, stdout, argc > 1 ? argv[1] : NULL);
return 0;
}
I am trying to create an MP4 file using libavcodec. I am using a raspberry pi which has a built in hardware H264 encoder. It outputs Annex B H264 frames and I am trying to see the proper way to save these frames into an MP4 container.
My first attempt simply wrote the MP4 header without building the extradata. The raspberry pi transmits as first frame the SPS and PPS info. This is followed by IDR and then the remaining H264 frames. I started with avformat_write_header and then repackaged the succeeding frames in AVPacket and used
av_write_frame(outputFormatCtx, &pkt);
This works fine but mplayer tries to decode the first frame ( the one containing SPS and PPS info ) and fails with decoding that frame. However, succeeding frames are decodable and the video plays fine from that point on.
I wanted to construct a proper MP4 file so I wanted the SPS and PPS information to go the MP4 header. I read that it should be in the avc1 atom and that I needed to build the extradata and somehow link it to the outputformatctx.
This is my effort so far, after parsing sps and pps from the returned encoder buffers. (I removed the leading 0x0000001 nal delimiters prior to memcpying to sps and pps).
if ((sps) && (pps)) {
//length of extradata is 6 bytes + 2 bytes for spslen + sps + 1 byte number of pps + 2 bytes for ppslen + pps
uint32_t extradata_len = 8 + spslen + 1 + 2 + ppslen;
outputStream->codecpar->extradata = (uint8_t*)av_mallocz(extradata_len);
outputStream->codecpar->extradata_size = extradata_len;
//start writing avcc extradata
outputStream->codecpar->extradata[0] = 0x01; //version
outputStream->codecpar->extradata[1] = sps[1]; //profile
outputStream->codecpar->extradata[2] = sps[2]; //comatibility
outputStream->codecpar->extradata[3] = sps[3]; //level
outputStream->codecpar->extradata[4] = 0xFC | 3; // reserved (6 bits), NALU length size - 1 (2 bits) which is 3
outputStream->codecpar->extradata[5] = 0xE0 | 1; // reserved (3 bits), num of SPS (5 bits) which is 1 sps
//write sps length
memcpy(&outputStream->codecpar->extradata[6],&spslen,2);
//Check to see if written correctly
uint16_t *cspslen=(uint16_t *)&outputStream->codecpar->extradata[6];
fprintf(stderr,"SPS length Wrote %d and read %d \n",spslen,*cspslen);
//Write the actual sps
int i = 0;
for (i=0; i<spslen; i++) {
outputStream->codecpar->extradata[8 + i] = sps[i];
}
for (size_t i = 0; i != outputStream->codecpar->extradata_size; ++i)
fprintf(stderr, "\\%02x", (unsigned char)outputStream->codecpar->extradata[i]);
fprintf(stderr,"\n");
//Number of pps
outputStream->codecpar->extradata[8 + spslen] = 0x01;
//Size of pps
memcpy(&outputStream->codecpar->extradata[8+spslen+1],&ppslen,2);
for (size_t i = 0; i != outputStream->codecpar->extradata_size; ++i)
fprintf(stderr, "\\%02x", (unsigned char)outputStream->codecpar->extradata[i]);
fprintf(stderr,"\n");
//Check to see if written correctly
uint16_t *cppslen=(uint16_t *)&outputStream->codecpar->extradata[+8+spslen+1];
fprintf(stderr,"PPS length Wrote %d and read %d \n",ppslen,*cppslen);
//Write actual PPS
for (i=0; i<ppslen; i++) {
outputStream->codecpar->extradata[8 + spslen + 1 + 2 + i] = pps[i];
}
//Output the extradata to check
for (size_t i = 0; i != outputStream->codecpar->extradata_size; ++i)
fprintf(stderr, "\\%02x", (unsigned char)outputStream->codecpar->extradata[i]);
fprintf(stderr,"\n");
//Access the outputFormatCtx internal AVCodecContext and copy the codecpar to it
AVCodecContext *avctx= outputFormatCtx->streams[0]->codec;
fprintf(stderr,"Extradata size output stream sps pps %d\n",outputStream->codecpar->extradata_size);
if(avcodec_parameters_to_context(avctx, outputStream->codecpar) < 0 ){
fprintf(stderr,"Error avcodec_parameters_to_context");
}
//Check to see if extradata was actually transferred to OutputformatCtx internal AVCodecContext
fprintf(stderr,"Extradata size after sps pps %d\n",avctx->extradata_size);
//Write the MP4 header
if(avformat_write_header(outputFormatCtx , NULL) < 0){
fprintf(stderr,"Error avformat_write_header");
ret = 1;
} else {
extradata_written=true;
fprintf(stderr,"EXTRADATA written\n");
}
}
The resulting video file does not play. The extradata is actually stored in the tail section of the MP4 file instead of the location in the MP4 header for avc1. So it is being written by libavcodec but written likely by avformat_write_trailer.
I will post the PPS and SPS info here and the final extradata byte string just in case the error was in forming the extradata.
Here is the buffer from the hardware encoder with sps and pps preceded by the nal delimiter
\00\00\00\01\27\64\00\28\ac\2b\40\a0\cd\00\f1\22\6a\00\00\00\01\28\ee\04\f2\c0
Here is the 13 byte sps:
27640028ac2b40a0cd00f1226a
Here is the 5 byte pps:
28ee04f2c0
Here is the final extradata byte string which is 29 bytes long. I hope I wrote the PPS and SPS size correctly.
\01\64\00\28\ff\e1\0d\00\27\64\00\28\ac\2b\40\a0\cd\00\f1\22\6a\01\05\00\28\ee\04\f2\c0
I did the same conversion from NAL delimiter 0x0000001 to 4 byte NAL size for the succeeding frames from the encoder and saved them to the file sequentially and then wrote the trailer.
Any idea where the mistake is? How can I write the extradata to its proper location in the MP4 header?
Thanks,
Chris
Well, I found the problem. The raspberry pi is little endian so I assumed that I must write the sps length and pps length and each NALU size in little endian. They need to be written in big endian. After I made the change, the avcc atom showed in mp4info and mplayer can now playback the video.
It's not necessary to access the outputformatctx internal avcodeccontext and modify it.
This post was very helpful:
Possible Locations for Sequence/Picture Parameter Set(s) for H.264 Stream
Thanks,
Chris
I'm using AudioToolbox to access m4a audio files with following code:
UInt32 packetsToRead = 1; //Does it makes difference?
void *buffer = malloc(maxPacketSize * packetsToRead);
for (UInt64 packetIndex = 0; packetIndex < packetCount; packetIndex++)
{
ioNumberOfPackets = packetsToRead;
ioNumberOfBytes = maxPacketSize * ioNumberOfPackets;
AudioFileReadPacketData(audioFile, NO, &ioNumbersOfBytes, NULL, packetIndex, &ioNumberOFPackets, buffer);
for (UInt32 batchPacketIndex = 0; batchPacketIndex < ioNumberOfPackets; batchPacketIndex++)
{
//What to do here to get amplitude value? How to get sample value?
}
packetIndex+=ioNumberOfPackets;
}
My audio format is:
AppleM4A, 8000 Hz, 16 Bit, 4096 frames per packet
The solution was to use extended audio file services. You just have to set up transition between client format and PCM. Got the right way overthere Audio Processing: Playing with volume level.
To get waveform data, you may first need to convert your compressed audio file into raw PCM samples, such as found inside a WAV file, or other non-compressed audio format. Try AVAssetReader, et.al.
Do you know the format in which GNU Radio ( File Sink in GNU Radio Companion) stores the samples in the Binary File?
I need to read these samples in Matlab, but the problem is the file is too big to be read in Matlab.
I am writing the program in C++ to read this binary file.
The file sink is just a dump of the data stream. If the data stream content was simple bytes then the content of the file is straightforward. If the data stream contained complex numbers then the file will contain a list of complex numbers where each complex number is given by two floats and each float by (usually) 4 bytes.
See the files gnuradio/gnuradio-core/src/lib/io/gr_file_sink.cc and gr_file_source.cc for the implementations of the gnuradio file reading and writing blocks.
You could also use python and gnuradio to convert the files into some other format.
from gnuradio import gr
# Assuming the data stream was complex numbers.
src = gr.file_source(gr.sizeof_gr_complex, "the_file_name")
snk = gr.vector_sink_c()
tb = gr.top_block()
tb.connect(src, snk)
tb.run()
# The complex numbers are then accessible as a python list.
data = snk.data()
Ben's answer still stands – but it's from a time long past (the module organization points at GNU Radio 3.6, I think). Organizationally, things are different now; data-wise, the File Sink remained the same.
GNU Radio now has relatively much block documentation in their wiki. In particular, the File Sink documentation page has a section on Handling File Sink data; not to overquote that:
// This is C++17
#include <algorithm>
#include <cmath>
#include <complex>
#include <cstddef>
#include <filesystem>
#include <fstream>
#include <string_view>
#include <vector>
#include <fmt/format.h>
#include <fmt/ranges.h>
using sample_t = std::complex<float>;
using power_t = float;
constexpr std::size_t read_block_size = 1 << 16;
int main(int argc, char *argv[]) {
// expect exactly one argument, a file name
if (argc != 2) {
fmt::print(stderr, "Usage: {} FILE_NAME", argv[0]);
return -1;
}
// just for convenience; we could as well just use `argv[1]` throughout the
// code
std::string_view filename(argv[1]);
// check whether file exists
if (!std::filesystem::exists(filename.data())) {
fmt::print(stderr, "file '{:s}' not found\n", filename);
return -2;
}
// calculate how many samples to read
auto file_size = std::filesystem::file_size(std::filesystem::path(filename));
auto samples_to_read = file_size / sizeof(sample_t);
// construct and reserve container for resulting powers
std::vector<power_t> powers;
powers.reserve(samples_to_read);
std::ifstream input_file(filename.data(), std::ios_base::binary);
if (!input_file) {
fmt::print(stderr, "error opening '{:s}'\n", filename);
return -3;
}
// construct and reserve container for read samples
// if read_block_size == 0, then read the whole file at once
std::vector<sample_t> samples;
if (read_block_size)
samples.resize(read_block_size);
else
samples.resize(samples_to_read);
fmt::print(stderr, "Reading {:d} samples…\n", samples_to_read);
while (samples_to_read) {
auto read_now = std::min(samples_to_read, samples.size());
input_file.read(reinterpret_cast<char *>(samples.data()),
read_now * sizeof(sample_t));
for (size_t idx = 0; idx < read_now; ++idx) {
auto magnitude = std::abs(samples[idx]);
powers.push_back(magnitude * magnitude);
}
samples_to_read -= read_now;
}
// we're not actually doing anything with the data. Let's print it!
fmt::print("Power\n{}\n", fmt::join(powers, "\n"));
}
mod_deflate always sends gzip data when the request header Accept-Encoding is gip, deflate.
How can I tell mod_deflate to prefer to send deflate (NOT zlib) instead of gzip?
If this isn't possible...why would the develpers name the module mod_deflate when it can't deflate. Also, what is the best way, if any, for me to submit a bug report to have this fixed in future releases?
After looking at the source code for mod_deflate I have come to the conclusion that it is impossible to send anything other than gzip.
Now, I'm not a c programmer and I don't think I'll be able to commit any patches myself...but from the source I can see that there are a few things that need to be fixed (warning, I've never written any c...so this is all probably terribly wrong)
/* add this method */
static const char *deflate_set_preferred_method(cmd_parms *cmd, void *dummy,
const char *arg1)
{
deflate_filter_config *c = ap_get_module_config(cmd->server->module_config,
&deflate_module);
if (arg2 != NULL && (!strcasecmp(arg1, "deflate") || !strcasecmp(arg1, "gzip") || !strcasecmp(arg1, "zlib") ) ) {
c->preferred_method = apr_pstrdup(cmd->pool, arg1);
}
else {
return apr_psprintf(cmd->pool, "Unknown preferred method type %s", arg1);
}
return NULL;
}
/* update some code to define "preferred_method" */
/*
Update all code that references the string "gzip" to take
into account "deflate", and "zlib" as well.
This is the part I really have no clue how to do.
lines: 539, 604, 607, 616, and 624 should be updates
line 624 could read something like this: */
if( !strcasecmp(preferred_method,"gzip") ){
/* add immortal gzip header */
e = apr_bucket_immortal_create(gzip_header, sizeof gzip_header,
f->c->bucket_alloc);
APR_BRIGADE_INSERT_TAIL(ctx->bb, e);
}
else if( !strcasecmp(preferred_method, "zlib") ){
/* do something to add the zlib headers here */
}
/* update this method */
static const command_rec deflate_filter_cmds[] = {
AP_INIT_TAKE12("DeflateFilterNote", deflate_set_note, NULL, RSRC_CONF,
"Set a note to report on compression ratio"),
AP_INIT_TAKE1("DeflateWindowSize", deflate_set_window_size, NULL,
RSRC_CONF, "Set the Deflate window size (1-15)"),
AP_INIT_TAKE1("DeflateBufferSize", deflate_set_buffer_size, NULL, RSRC_CONF,
"Set the Deflate Buffer Size"),
AP_INIT_TAKE1("DeflateMemLevel", deflate_set_memlevel, NULL, RSRC_CONF,
"Set the Deflate Memory Level (1-9)"),
AP_INIT_TAKE1("DeflateCompressionLevel", deflate_set_compressionlevel, NULL, RSRC_CONF,
"Set the Deflate Compression Level (1-9)"),
AP_INIT_TAKE1("DeflatePreferredMethod", deflate_set_preferred_method, NULL, RSRC_CONF,
"Set the Preferred Compression Method: deflate, gzip, or zlib (not-recommended)"),
{NULL}
};