Commit c6b67b6a authored by Gencer Sumbul's avatar Gencer Sumbul
Browse files

adding files

parent 1c1e8776
......@@ -31,7 +31,6 @@ class Retrieval():
self.logger.info('Retrieval is finished')
def feature_extraction(self):
# Get the configuration.
self.logger.info('feature extraction is started')
if (not os.path.isfile(self.query_feat_path)) or (not os.path.isfile(self.archive_feat_path)):
self.prep_feature_extraction()
......@@ -157,7 +156,6 @@ class Retrieval():
retrieved_labels.append(retrieved[i])
else:
break
retrieved_labels = np.array(retrieved_labels)
normalized_discounted_cumulative_gains = np.zeros(max_topk)
discounted_cumulative_gains = np.zeros(max_topk)
......@@ -199,18 +197,18 @@ class Retrieval():
with timer_calc() as elapsed_time:
scores = np.array(ray.get(result_ids))
ray.shutdown()
normalized_discounted_cumulative_gains += np.sum(scores[:,0], axis=0)#normalized_discounted_cumulative_gains
average_precision += np.sum(scores[:,1], axis=0)#average_precision
normalized_discounted_cumulative_gains += np.sum(scores[:,0], axis=0)
average_precision += np.sum(scores[:,1], axis=0)
self.logger.info('{} tasks finished within {:0.2f} seconds'.format(len(result_ids), elapsed_time()))
result_ids = []
ray.init(num_cpus=num_cpus, object_store_memory = 100 * 1024 * 1024 * 1024)#, memory = 100 * 1024 * 1024 * 1024)
ray.init(num_cpus=num_cpus, object_store_memory = 100 * 1024 * 1024 * 1024)
if not len(result_ids) == 0:
with timer_calc() as elapsed_time:
scores = np.array(ray.get(result_ids))
ray.shutdown()
normalized_discounted_cumulative_gains += np.sum(scores[:,0], axis=0)#normalized_discounted_cumulative_gains
average_precision += np.sum(scores[:,1], axis=0)#average_precision
normalized_discounted_cumulative_gains += np.sum(scores[:,0], axis=0)
average_precision += np.sum(scores[:,1], axis=0)
self.logger.info('{} tasks finished within {:0.2f} seconds'.format(len(result_ids), elapsed_time()))
result_ids = []
......@@ -221,6 +219,4 @@ class Retrieval():
hf.create_dataset('average_precision', data=average_precision)
def restore_weigths(self):
# self.model._set_inputs(self.dataset.element_spec[0])
self.model.neural_net = tf.keras.models.load_model(os.path.join(self.config.dumps.model_weights, self.config.suffix))
#self.model.load_model(os.path.join(self.config.dumps.model_weights, self.config.suffix))
\ No newline at end of file
......@@ -5,60 +5,24 @@ class BatchHardTripletLoss(tf.keras.losses.Loss):
super(BatchHardTripletLoss, self).__init__(reduction=tf.keras.losses.Reduction.NONE)
@tf.function
def _pairwise_distances(self, embeddings, squared=False):
"""Compute the 2D matrix of distances between all the embeddings.
Args:
embeddings: tensor of shape (batch_size, embed_dim)
squared: Boolean. If true, output is the pairwise squared euclidean distance matrix.
If false, output is the pairwise euclidean distance matrix.
Returns:
pairwise_distances: tensor of shape (batch_size, batch_size)
"""
# Get the dot product between all embeddings
# shape (batch_size, batch_size)
def get_distances(self, embeddings):
dot_product = tf.matmul(embeddings, tf.transpose(embeddings))
# Get squared L2 norm for each embedding. We can just take the diagonal of `dot_product`.
# This also provides more numerical stability (the diagonal of the result will be exactly 0).
# shape (batch_size,)
square_norm = tf.linalg.diag_part(dot_product)
# Compute the pairwise distance matrix as we have:
# ||a - b||^2 = ||a||^2 - 2 <a, b> + ||b||^2
# shape (batch_size, batch_size)
distances = tf.expand_dims(square_norm, 0) - 2.0 * dot_product + tf.expand_dims(square_norm, 1)
# Because of computation errors, some distances might be negative so we put everything >= 0.0
distances = tf.maximum(distances, 0.0)
if not squared:
# Because the gradient of sqrt is infinite when distances == 0.0 (ex: on the diagonal)
# we need to add a small epsilon where distances == 0.0
mask = tf.cast(tf.equal(distances, 0.0), tf.float32)
distances = distances + mask * 1e-16
distances = tf.sqrt(distances)
# Correct the epsilon added: set the distances on the mask to be exactly 0.0
distances = distances * (1.0 - mask)
mask = tf.cast(tf.equal(distances, 0.0), tf.float32)
distances = distances + mask * 1e-16
distances = tf.sqrt(distances)
distances = distances * (1.0 - mask)
return distances
@tf.function
def _get_anchor_positive_triplet_mask(self, labels):
"""Return a 2D mask where mask[a, p] is True iff a and p are distinct and have same label.
Args:
labels: tf.int32 `Tensor` with shape [batch_size]
Returns:
mask: tf.bool `Tensor` with shape [batch_size, batch_size]
"""
# Check that i and j are distinct
indices_equal = tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool)
indices_not_equal = tf.math.logical_not(indices_equal)
# Check if labels[i] == labels[j]
# Uses broadcasting where the 1st argument has shape (1, batch_size, 43) and the 2nd (batch_size, 1, 43)
def get_anchor_positive_mask(self, labels):
indices_not_equal = tf.math.logical_not(tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool))
labels_equal = tf.greater(
tf.divide(
tf.reduce_sum(
......@@ -76,21 +40,11 @@ class BatchHardTripletLoss(tf.keras.losses.Loss):
tf.ones(dtype=tf.float32, shape=[tf.shape(labels)[0], tf.shape(labels)[0]]) * 0.5
)
# Combine the two masks
mask = tf.logical_and(indices_not_equal, labels_equal)
return mask
@tf.function
def _get_anchor_negative_triplet_mask(self, labels):
"""Return a 2D mask where mask[a, n] is True iff a and n have distinct labels.
Args:
labels: tf.int32 `Tensor` with shape [batch_size]
Returns:
mask: tf.bool `Tensor` with shape [batch_size, batch_size]
"""
# Check if labels[i] != labels[k]
# Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1)
def get_anchor_negative_mask(self, labels):
labels_not_equal = tf.less(
tf.divide(
tf.reduce_sum(
......@@ -112,51 +66,23 @@ class BatchHardTripletLoss(tf.keras.losses.Loss):
return mask
@tf.function
def call(self, labels, embeddings, margin=1.0, squared=False):
"""Build the triplet loss over a batch of embeddings.
For each anchor, we get the hardest positive and hardest negative to form a triplet.
Args:
labels: labels of the batch, of size (batch_size,)
embeddings: tensor of shape (batch_size, embed_dim)
margin: margin for triplet loss
squared: Boolean. If true, output is the pairwise squared euclidean distance matrix.
If false, output is the pairwise euclidean distance matrix.
Returns:
triplet_loss: scalar tensor containing the triplet loss
"""
# Get the pairwise distance matrix
pairwise_dist = self._pairwise_distances(embeddings, squared=squared)
# For each anchor, get the hardest positive
# First, we need to get a mask for every valid positive (they should have same label)
mask_anchor_positive = self._get_anchor_positive_triplet_mask(labels)
mask_anchor_positive = tf.cast(mask_anchor_positive, tf.float32)
# We put to 0 any element where (a, p) is not valid (valid if a != p and label(a) == label(p))
anchor_positive_dist = tf.multiply(mask_anchor_positive, pairwise_dist)
# shape (batch_size, 1)
hardest_positive_dist = tf.reduce_max(anchor_positive_dist, axis=1, keepdims=True)
d_pos = hardest_positive_dist
# For each anchor, get the hardest negative
# First, we need to get a mask for every valid negative (they should have different labels)
mask_anchor_negative = self._get_anchor_negative_triplet_mask(labels)
mask_anchor_negative = tf.cast(mask_anchor_negative, tf.float32)
# We add the maximum value in each row to the invalid negatives (label(a) == label(n))
max_anchor_negative_dist = tf.reduce_max(pairwise_dist, axis=1, keepdims=True)
anchor_negative_dist = pairwise_dist + max_anchor_negative_dist * (1.0 - mask_anchor_negative)
# shape (batch_size,)
hardest_negative_dist = tf.reduce_min(anchor_negative_dist, axis=1, keepdims=True)
d_neg = hardest_negative_dist
# Combine biggest d(a, p) and smallest d(a, n) into final triplet loss
def call(self, labels, embeddings, margin=1.0):
pairwise_dist = self.get_distances(embeddings)
positive_mask = self.get_anchor_positive_mask(labels)
positive_mask = tf.cast(positive_mask, tf.float32)
positive_distance = tf.multiply(positive_mask, pairwise_dist)
hardest_positive_dist = tf.reduce_max(positive_distance, axis=1, keepdims=True)
negative_mask = self.get_anchor_negative_mask(labels)
negative_mask = tf.cast(negative_mask, tf.float32)
max_negative_dist = tf.reduce_max(pairwise_dist, axis=1, keepdims=True)
negative_distance = pairwise_dist + max_negative_dist * (1.0 - negative_mask)
hardest_negative_dist = tf.reduce_min(negative_distance, axis=1, keepdims=True)
triplet_loss = tf.maximum(hardest_positive_dist - hardest_negative_dist + margin, 0.0)
# Get final mean triplet loss
triplet_loss = tf.reduce_mean(triplet_loss)
return triplet_loss
\ No newline at end of file
......@@ -7,19 +7,6 @@ class ContrastiveLoss(tf.keras.losses.Loss):
# Based on https://github.com/google-research/simclr/tree/master/tf2
@tf.function
def __call__(self, labels, hidden, hidden_norm=True, temperature=1.0, weights=1.0):
"""Compute loss for model.
Args:
hidden: hidden vector (`Tensor`) of shape (2 * bsz, dim).
hidden_norm: whether or not to use normalization on the hidden vector.
temperature: a `floating` number for temperature scaling.
tpu_context: context information for tpu.
weights: a weighting number or vector.
Returns:
A loss scalar.
The logits for contrastive prediction task.
The labels for contrastive prediction task.
"""
# Get (normalized) hidden1 and hidden2.
if hidden_norm:
hidden = tf.math.l2_normalize(hidden, -1)
hidden1, hidden2 = tf.split(hidden, 2, 0)
......
......@@ -35,13 +35,5 @@ class StabilityLoss(tf.keras.losses.Loss):
-1.0+tf.keras.backend.epsilon(), 1.0-tf.keras.backend.epsilon()
)
)
# stability_loss += tf.math.abs(tf.math.acos(
# tf.clip_by_value(
# tf.reduce_sum(
# tf.multiply(grads_norm[i], grads_norm[i+1]),
# ),
# -1.0+tf.keras.backend.epsilon(), 1.0-tf.keras.backend.epsilon()
# )
# ) - (pi / 2.))
return stability_loss / (len(grads_norm) - 1)
\ No newline at end of file
......@@ -47,7 +47,7 @@ class Model():
self.trackers['stability_loss'](stability_loss)
feat_grads[:] = []
else:
feature_grad = gradients[tvars_dict['feature_idx']] #plas_gradients[0] #
feature_grad = gradients[tvars_dict['feature_idx']]
feat_grads.append(tf.reshape(feature_grad, [-1]))
del stability_tape
task.train_loss_tracker(task_loss)
......@@ -79,7 +79,7 @@ class Model():
self.trackers['stability_loss'](stability_loss)
feat_grads[:] = []
else:
feature_grad = gradients[tvars_dict['feature_idx']] #plas_gradients[0] #
feature_grad = gradients[tvars_dict['feature_idx']]
feat_grads.append(tf.reshape(feature_grad, [-1]))
del plasticity_tape
del stability_tape
......@@ -127,22 +127,22 @@ class Model():
def validation_step(self, batch):
loss = 0.
out_std = self.neural_net(batch['img'], training=False)
for task_id, task in enumerate(self.tasks):
for task in self.tasks:
if not task.input_key == 'img':
out = self.neural_net(batch[task.input_key], training=False)
else:
out = out_std
task_loss = task.loss_fnc(batch[task.ground_truth], out['{}_out'.format(task.name)]) #TODO [task_id+1]
task_loss = task.loss_fnc(batch[task.ground_truth], out['{}_out'.format(task.name)])
loss += task_loss
task.val_loss_tracker(task_loss)
self.trackers['joint_val_loss'](loss)
def validation(self, val_dataset):
for batch_id, batch_raw in enumerate(val_dataset):
for batch_raw in val_dataset:
self.prep_batch(batch_raw, training=False)
self.validation_step(self.batch)
def prep_batch(self, batch, training):
def prep_batch(self, batch, training):
def augmentation(img):
if training:
if self.config.dataset == 'DLRSD':
......@@ -195,7 +195,7 @@ class Model():
self.task_heads = []
self.tasks = tasks
with tf.python.keras.backend.get_graph().as_default() as g:
with tf.name_scope("backbone") as scope:
with tf.name_scope("backbone"):
self.backbone = tf.keras.applications.DenseNet121(
weights=None, include_top=False, pooling=None, input_shape=input_shape)
self.pooling = tf.keras.layers.GlobalAveragePooling2D()
......@@ -208,9 +208,9 @@ class Model():
out_dict['feature'] = feature
out_dict['out_backbone'] = out_backbone
for task_id, task in enumerate(self.tasks):
with tf.name_scope(task.name) as scope:
with tf.name_scope(task.name):
task_out = self.task_heads[task_id](out_backbone, training)
out_dict['{}_embed'.format(task.name)] = task_out['{}_embed'.format(task.name)] #self.l2_normalize(task_out['{}_embed'.format(task.name)])
out_dict['{}_embed'.format(task.name)] = task_out['{}_embed'.format(task.name)]
out_dict['{}_out'.format(task.name)] = task_out['{}_out'.format(task.name)]
return out_dict
\ No newline at end of file
......@@ -43,11 +43,11 @@ class SupCoOccurencePred(Task):
super(SupCoOccurencePred.TaskHead, self).__init__(**kwargs)
self.pooling = tf.keras.layers.GlobalAveragePooling2D()
self.embed_dim = config.embed_dim
self.SupCoOccurencePred_embedding = tf.keras.layers.Dense(
self.embedding = tf.keras.layers.Dense(
units=self.embed_dim,
activation=tf.keras.activations.relu
)
self.SupCoOccurencePred_head = tf.keras.layers.Dense(
self.pred_head = tf.keras.layers.Dense(
units=17*17,
activation=tf.keras.activations.relu
)
......@@ -55,6 +55,6 @@ class SupCoOccurencePred(Task):
def call(self, inputs, training):
feature = self.pooling(inputs)
with tf.name_scope('embed'):
SupCoOccurencePred_emb = self.SupCoOccurencePred_embedding(feature, training=training)
SupCoOccurencePred_logits = self.SupCoOccurencePred_head(SupCoOccurencePred_emb, training=training)
return {'SupCoOccurencePred_embed': SupCoOccurencePred_emb, 'SupCoOccurencePred_out': SupCoOccurencePred_logits}
\ No newline at end of file
embed = self.embedding(feature, training=training)
logits = self.pred_head(embed, training=training)
return {'SupCoOccurencePred_embed': embed, 'SupCoOccurencePred_out': logits}
\ No newline at end of file
......@@ -13,11 +13,11 @@ class SupSceneCls(Task):
super(SupSceneCls.TaskHead, self).__init__(**kwargs)
self.pooling = tf.keras.layers.GlobalAveragePooling2D()
self.embed_dim = config.embed_dim
self.SupSceneCls_embedding = tf.keras.layers.Dense(
self.embedding = tf.keras.layers.Dense(
units=self.embed_dim,
activation=tf.keras.activations.relu
)
self.SupSceneCls_head = tf.keras.layers.Dense(
self.pred_head = tf.keras.layers.Dense(
units=config.nb_class,
activation=None
)
......@@ -25,6 +25,6 @@ class SupSceneCls(Task):
def call(self, inputs, training):
feature = self.pooling(inputs)
with tf.name_scope('embed'):
SupSceneCls_emb = self.SupSceneCls_embedding(feature, training=training)
SupSceneCls_logits = self.SupSceneCls_head(SupSceneCls_emb, training=training)
return {'SupSceneCls_embed': SupSceneCls_emb, 'SupSceneCls_out': SupSceneCls_logits}
\ No newline at end of file
embed = self.embedding(feature, training=training)
logits = self.pred_head(embed, training=training)
return {'SupSceneCls_embed': embed, 'SupSceneCls_out': logits}
\ No newline at end of file
......@@ -13,7 +13,7 @@ class SupSimLearning(Task):
super(SupSimLearning.TaskHead, self).__init__(**kwargs)
self.pooling = tf.keras.layers.GlobalAveragePooling2D()
self.embed_dim = config.embed_dim
self.SupSimLearning_embedding = tf.keras.layers.Dense(
self.embedding = tf.keras.layers.Dense(
units=self.embed_dim,
activation=tf.keras.activations.relu
)
......@@ -21,5 +21,5 @@ class SupSimLearning(Task):
def call(self, inputs, training):
feature = self.pooling(inputs)
with tf.name_scope('embed'):
SupSimLearning_emb = self.SupSimLearning_embedding(feature, training=training)
return {'SupSimLearning_embed':SupSimLearning_emb, 'SupSimLearning_out':SupSimLearning_emb}
\ No newline at end of file
embed = self.embedding(feature, training=training)
return {'SupSimLearning_embed':embed, 'SupSimLearning_out':embed}
\ No newline at end of file
......@@ -40,7 +40,7 @@ class UnsupSimLearning(Task):
super(UnsupSimLearning.TaskHead, self).__init__(**kwargs)
self.pooling = tf.keras.layers.GlobalAveragePooling2D()
self.embed_dim = config.embed_dim
self.UnsupSimLearning_embedding = tf.keras.layers.Dense(
self.embedding = tf.keras.layers.Dense(
units=self.embed_dim,
activation=tf.keras.activations.relu
)
......@@ -48,5 +48,5 @@ class UnsupSimLearning(Task):
def call(self, inputs, training):
feature = self.pooling(inputs)
with tf.name_scope('embed'):
UnsupSimLearning_embed = self.UnsupSimLearning_embedding(feature, training=training)
return {'UnsupSimLearning_embed': UnsupSimLearning_embed, 'UnsupSimLearning_out': UnsupSimLearning_embed}
\ No newline at end of file
embed = self.embedding(feature, training=training)
return {'UnsupSimLearning_embed': embed, 'UnsupSimLearning_out': embed}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment