I got code from Keras (I'm new to deep learning) for graph attention network (GAT) which I tweaked for binary classification and I added edge weights. The code is now:
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import numpy as np
import pandas as pd
import os
import warnings
warnings.filterwarnings("ignore")
pd.set_option("display.max_columns", 6)
pd.set_option("display.max_rows", 6)
np.random.seed(2)
citations = pd.read_csv("../data/CPDB_matrix.tsv", sep="\t", header=None,
names=["source", "target", "interaction_strength"])
citations = pd.concat([citations, citations.rename(columns={"target": "source", "source": "target"})]).drop_duplicates()
citations = pd.concat([citations, citations.rename(columns={"target": "source", "source": "target"})]).drop_duplicates()
papers = pd.read_csv(
"../data/cMCL.tsv",
sep="\t",
header=None,
names=["paper_id"] + [f"term_{idx}" for idx in range(89)] + ["subject"],
)
cancer_classes = {"cancer"} # Example: Modify with actual cancer-related labels
papers["subject"] = papers["subject"].apply(lambda x: 1 if x in cancer_classes else (0 if x == 'non_cancer' else -1)) # -1 for unknown
paper_idx = {name: idx for idx, name in enumerate(sorted(papers["paper_id"].unique()))}
papers["paper_id"] = papers["paper_id"].apply(lambda name: paper_idx[name])
citations["source"] = citations["source"].apply(lambda name: paper_idx[name])
citations["target"] = citations["target"].apply(lambda name: paper_idx[name])
random_indices = np.random.permutation(papers[papers["subject"] != -1].index)
train_data = papers.iloc[random_indices[: len(random_indices) // 2]]
test_data = papers.iloc[random_indices[len(random_indices) // 2 :]]
#train_data = valid_papers.iloc[random_indices[: len(random_indices) // 2]]
#test_data = valid_papers.iloc[random_indices[len(random_indices) // 2 :]]
train_indices = train_data["paper_id"].to_numpy()
test_indices = test_data["paper_id"].to_numpy()
train_labels = train_data["subject"].to_numpy()
test_labels = test_data["subject"].to_numpy()
edges = tf.convert_to_tensor(citations[["source", "target"]].values, dtype=tf.int32)
edge_weights = tf.convert_to_tensor(citations["interaction_strength"].values, dtype=tf.float32)
node_states = tf.convert_to_tensor(papers.sort_values("paper_id").iloc[:, 1:-1].to_numpy(), dtype=tf.float32)
train_node_states = tf.gather(node_states, train_indices)
test_node_states = tf.gather(node_states, test_indices)
reverse_paper_idx = {v: k for v,k in enumerate(papers['paper_id'])}
train_paper_ids = [reverse_paper_idx[idx] for idx in train_indices]
train_edges_mask = np.isin(citations["source"], train_paper_ids) & np.isin(citations["target"], train_paper_ids)
train_edges = citations[train_edges_mask][["source", "target"]].values
train_edge_weights = citations[train_edges_mask]["interaction_strength"].values
print("Train edges:\n", train_edges)
print("Train edge weights:\n", train_edge_weights)
class GraphAttention(layers.Layer):
def __init__(
self,
units,
kernel_initializer="glorot_uniform",
kernel_regularizer=None,
**kwargs,
):
super().__init__(**kwargs)
self.units = units
self.kernel_initializer = keras.initializers.get(kernel_initializer)
self.kernel_regularizer = keras.regularizers.get(kernel_regularizer)
def build(self, input_shape):
self.kernel = self.add_weight(
shape=(input_shape[0][-1], self.units),
trainable=True,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
name="kernel",
)
self.kernel_attention = self.add_weight(
shape=(self.units * 2, 1),
trainable=True,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
name="kernel_attention",
)
self.built = True
def call(self, inputs):
node_states, edges, edge_weights = inputs
#node_states, edges = inputs
# Transform node states
node_states_transformed = tf.matmul(node_states, self.kernel)
# Compute attention scores
node_states_expanded = tf.gather(node_states_transformed, edges)
node_states_expanded = tf.reshape(node_states_expanded, (tf.shape(edges)[0], -1))
attention_scores = tf.nn.leaky_relu(tf.matmul(node_states_expanded, self.kernel_attention))
attention_scores = tf.squeeze(attention_scores, -1)
# Apply edge weights
attention_scores *= edge_weights # Multiply attention by interaction strength
# Normalize scores
attention_scores = tf.math.exp(tf.clip_by_value(attention_scores, -2, 2))
attention_scores_sum = tf.math.unsorted_segment_sum(
data=attention_scores,
segment_ids=edges[:, 0],
num_segments=tf.reduce_max(edges[:, 0]) + 1,
)
attention_scores_norm = attention_scores / tf.repeat(attention_scores_sum,
tf.math.bincount(tf.cast(edges[:, 0], "int32")))
# Aggregate neighbor states
node_states_neighbors = tf.gather(node_states_transformed, edges[:, 1])
out = tf.math.unsorted_segment_sum(
data=node_states_neighbors * attention_scores_norm[:, tf.newaxis],
segment_ids=edges[:, 0],
num_segments=tf.shape(node_states)[0],
)
return out
class MultiHeadGraphAttention(layers.Layer):
def __init__(self, units, num_heads=8, merge_type="concat", **kwargs):
super().__init__(**kwargs)
self.num_heads = num_heads
self.merge_type = merge_type
self.attention_layers = [GraphAttention(units) for _ in range(num_heads)]
def call(self, inputs):
atom_features, pair_indices, edge_weights = inputs
# Obtain outputs from each attention head
outputs = [
attention_layer([atom_features, pair_indices, edge_weights])
for attention_layer in self.attention_layers
]
# Concatenate or average the node states from each head
if self.merge_type == "concat":
outputs = tf.concat(outputs, axis=-1)
else:
outputs = tf.reduce_mean(tf.stack(outputs, axis=-1), axis=-1)
# Activate and return node states
return tf.nn.relu(outputs)
class GraphAttentionNetwork(keras.Model):
def __init__(
self,
node_states,
edges,
edge_weights,
hidden_units,
num_heads,
num_layers,
output_dim,
**kwargs,
):
super().__init__(**kwargs)
self.node_states = node_states
self.edges = edges
self.edge_weights = edge_weights
self.preprocess = layers.Dense(hidden_units * num_heads, activation="relu")
self.attention_layers = [
MultiHeadGraphAttention(hidden_units, num_heads) for _ in range(num_layers)
]
self.output_layer = layers.Dense(output_dim)
def call(self, inputs):
node_states, edges, edge_weights = inputs
x = self.preprocess(node_states)
for attention_layer in self.attention_layers:
x = attention_layer([x, edges, edge_weights]) + x
outputs = self.output_layer(x)
return outputs
def train_step(self, data):
indices, labels = data
with tf.GradientTape() as tape:
# Forward pass
outputs = self([self.node_states, self.edges, self.edge_weights])
# Compute loss
a, indices_tensor, c = indices # Extract the actual indices tensor
#tf.cast(indices_tensor, tf.float32)
#indices = (a, indices_tensor, c)
#indices = tf.convert_to_tensor((a, tf.cast(indices_tensor, tf.float32), c))
print(indices)
loss = selfpiled_loss(labels, tf.gather(outputs, indices))
# Compute gradients
grads = tape.gradient(loss, self.trainable_weights)
# Apply gradients (update weights)
optimizer.apply_gradients(zip(grads, self.trainable_weights))
# Update metric(s)
selfpiled_metrics.update_state(labels, tf.gather(outputs, indices))
return {m.name: m.result() for m in self.metrics}
def predict_step(self, data):
indices = data
# Forward pass
outputs = self([self.node_states, self.edges, self.edge_weights])
# Compute probabilities
return tf.nn.softmax(tf.gather(outputs, indices))
def test_step(self, data):
indices, labels = data
# Forward pass
outputs = self([self.node_states, self.edges, edge_weights])
# Compute loss
loss = selfpiled_loss(labels, tf.gather(outputs, indices))
# Update metric(s)
selfpiled_metrics.update_state(labels, tf.gather(outputs, indices))
return {m.name: m.result() for m in self.metrics}
# Define hyper-parameters
HIDDEN_UNITS = 100
NUM_HEADS = 8
NUM_LAYERS = 3
OUTPUT_DIM = len(class_values)
NUM_EPOCHS = 100
BATCH_SIZE = 256
VALIDATION_SPLIT = 0.1
LEARNING_RATE = 3e-1
MOMENTUM = 0.9
loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = keras.optimizers.SGD(LEARNING_RATE, momentum=MOMENTUM)
accuracy_fn = keras.metrics.SparseCategoricalAccuracy(name="acc")
early_stopping = keras.callbacks.EarlyStopping(
monitor="val_acc", min_delta=1e-5, patience=5, restore_best_weights=True
)
gat_model = GraphAttentionNetwork(
node_states, edges, edge_weights, HIDDEN_UNITS, NUM_HEADS, NUM_LAYERS, output_dim=2 # Binary output
)
# Compile model
gat_modelpile(loss=keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=keras.optimizers.Adam(learning_rate=1e-2),
metrics=[keras.metrics.BinaryAccuracy(name="acc")])
gat_model.fit(
x=[train_node_states, train_edges, train_edge_weights],
y=train_labels,
validation_split=0.1,
batch_size=256,
epochs=100,
callbacks=[keras.callbacks.EarlyStopping(monitor="val_acc", patience=5, restore_best_weights=True)],
verbose=2,
)
However, I keep getting the following error:
TypeError Traceback (most recent call last)
Cell In[1425], line 1
----> 1 gat_model.fit(
2 #x=train_indices,
3 x=[train_node_states, train_edges, train_edge_weights],
4 y=train_labels,
5 validation_split=0.1,
6 batch_size=256,
7 epochs=100,
8 callbacks=[keras.callbacks.EarlyStopping(monitor="val_acc", patience=5, restore_best_weights=True)],
9 verbose=2,
10 )
File /gpfs/apps/MN4/MINIFORGE/24.11.3-2/envs/maia-env/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
119 filtered_tb = _process_traceback_frames(e.__traceback__)
120 # To get the full stack trace, call:
121 # `keras.config.disable_traceback_filtering()`
--> 122 raise e.with_traceback(filtered_tb) from None
123 finally:
124 del filtered_tb
Cell In[1393], line 40, in GraphAttentionNetwork.train_step(self, data)
36 #tf.cast(indices_tensor, tf.float32)
37 #indices = (a, indices_tensor, c)
38 #indices = tf.convert_to_tensor((a, tf.cast(indices_tensor, tf.float32), c))
39 print(indices)
---> 40 loss = selfpiled_loss(labels, tf.gather(outputs, indices))
41 # Compute gradients
42 grads = tape.gradient(loss, self.trainable_weights)
TypeError: Cannot convert a list containing a tensor of dtype <dtype: 'int64'> to <dtype: 'float32'> (Tensor is: <tf.Tensor 'data_1:0' shape=(None, 2) dtype=int64>)
When I use x = train_indices, I get
AttributeError Traceback (most recent call last)
Cell In[2086], line 1
----> 1 gat_model.fit(
2 x=train_indices,
3 #x=[train_node_states, train_edges, train_edge_weights],
4 y=train_labels,
5 validation_split=0.1,
6 batch_size=256,
7 epochs=100,
8 callbacks=[keras.callbacks.EarlyStopping(monitor="val_acc", patience=5, restore_best_weights=True)],
9 verbose=2,
10 )
File /gpfs/apps/MN4/MINIFORGE/24.11.3-2/envs/maia-env/lib/python3.12/site-packages/keras/src/utils/traceback_utils.py:122, in filter_traceback.<locals>.error_handler(*args, **kwargs)
119 filtered_tb = _process_traceback_frames(e.__traceback__)
120 # To get the full stack trace, call:
121 # `keras.config.disable_traceback_filtering()`
--> 122 raise e.with_traceback(filtered_tb) from None
123 finally:
124 del filtered_tb
Cell In[2054], line 48, in GraphAttentionNetwork.train_step(self, data)
46 print(loss)
47 # Compute gradients
---> 48 grads = tape.gradient(loss, self.trainable_weights)
49 # Apply gradients (update weights)
50 optimizer.apply_gradients(zip(grads, self.trainable_weights))
AttributeError: 'SymbolicTensor' object has no attribute 'handle'
Any ideas? I tried a bunch of suggestions from chatGPT but nothing worked. Any thoughts would be much appreciated.